From 043c35bde50f5093ccd904b90cadebf7fd11f0f6 Mon Sep 17 00:00:00 2001 From: hltav Date: Fri, 5 Jun 2026 10:20:47 -0300 Subject: [PATCH 1/2] =?UTF-8?q?fix(jobstore,pipeline):=20corrige=20expira?= =?UTF-8?q?=C3=A7=C3=A3o=20prematura=20de=20=C3=ADndices=20e=20janela=20va?= =?UTF-8?q?zia=20durante=20reindexa=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problemas resolvidos: - scraper:jobs:index expirava em 72h, causando API vazia entre execuções semanais - jobTTL de 72h insuficiente para cobrir o intervalo de scraping de 7 dias - Del nos índices de keyword criava janela de vazio durante reindexação (até 40min) Mudanças em internal/jobstore/jobstore.go: - Remove Expire(indexKey) do pipeline por vaga — índice global agora é permanente - Aumenta jobTTL de 72h para 9 dias (> janela de 7 dias do filtro r604800) - IDs órfãos continuam sendo removidos organicamente pelo GetAll via SRem Mudanças em internal/pipeline/pipeline.go: - Substitui Del + reindexação direta por estratégia blue/green com chaves :next - Escreve novo índice em scraper:jobs:keyword::next, depois RENAME atômico - RENAME garante que clientes nunca vejam chave vazia durante reindexação - Aumenta indexTTL de 72h para 9 dias, alinhado com jobTTL - Índice global (scraper:jobs:index) permanece sem TTL --- .cspell/custom-dictionary-workspace.txt | 66 + backend/bruno/Jobs/SearchJobs.bru | 8 +- backend/bruno/environments/Local.bru | 2 +- scraper-go/go.mod | 10 +- scraper-go/go.sum | 12 +- scraper-go/internal/jobstore/jobstore.go | 3 +- scraper-go/internal/jobstore/jobstore_test.go | 179 + scraper-go/internal/pipeline/pipeline.go | 154 +- scraper-go/internal/pipeline/pipeline_test.go | 190 ++ .../alicebob/miniredis/v2/.gitignore | 6 + .../alicebob/miniredis/v2/CHANGELOG.md | 357 ++ .../github.com/alicebob/miniredis/v2/LICENSE | 21 + .../github.com/alicebob/miniredis/v2/Makefile | 33 + .../alicebob/miniredis/v2/README.md | 344 ++ .../github.com/alicebob/miniredis/v2/check.go | 63 + .../alicebob/miniredis/v2/cmd_client.go | 68 + .../alicebob/miniredis/v2/cmd_cluster.go | 122 + .../alicebob/miniredis/v2/cmd_command.go | 14 + .../alicebob/miniredis/v2/cmd_connection.go | 281 ++ .../alicebob/miniredis/v2/cmd_generic.go | 789 +++++ .../alicebob/miniredis/v2/cmd_geo.go | 577 ++++ .../alicebob/miniredis/v2/cmd_hash.go | 797 +++++ .../alicebob/miniredis/v2/cmd_hll.go | 71 + .../alicebob/miniredis/v2/cmd_info.go | 43 + .../alicebob/miniredis/v2/cmd_list.go | 931 +++++ .../alicebob/miniredis/v2/cmd_object.go | 50 + .../alicebob/miniredis/v2/cmd_pubsub.go | 254 ++ .../alicebob/miniredis/v2/cmd_scripting.go | 345 ++ .../alicebob/miniredis/v2/cmd_server.go | 153 + .../alicebob/miniredis/v2/cmd_set.go | 701 ++++ .../alicebob/miniredis/v2/cmd_sorted_set.go | 1857 ++++++++++ .../alicebob/miniredis/v2/cmd_stream.go | 1698 +++++++++ .../alicebob/miniredis/v2/cmd_string.go | 1243 +++++++ .../alicebob/miniredis/v2/cmd_transactions.go | 140 + .../github.com/alicebob/miniredis/v2/db.go | 824 +++++ .../alicebob/miniredis/v2/direct.go | 862 +++++ .../alicebob/miniredis/v2/fpconv/LICENSE.txt | 26 + .../alicebob/miniredis/v2/fpconv/Makefile | 6 + .../alicebob/miniredis/v2/fpconv/README.md | 3 + .../alicebob/miniredis/v2/fpconv/dtoa.go | 286 ++ .../alicebob/miniredis/v2/fpconv/fp.go | 96 + .../alicebob/miniredis/v2/fpconv/powers.go | 82 + .../github.com/alicebob/miniredis/v2/geo.go | 46 + .../alicebob/miniredis/v2/geohash/LICENSE | 22 + .../alicebob/miniredis/v2/geohash/README.md | 2 + .../alicebob/miniredis/v2/geohash/base32.go | 44 + .../alicebob/miniredis/v2/geohash/geohash.go | 269 ++ .../alicebob/miniredis/v2/gopher-json/LICENSE | 24 + .../miniredis/v2/gopher-json/README.md | 1 + .../alicebob/miniredis/v2/gopher-json/json.go | 189 + .../github.com/alicebob/miniredis/v2/hll.go | 42 + .../alicebob/miniredis/v2/hyperloglog/LICENSE | 21 + .../miniredis/v2/hyperloglog/README.md | 1 + .../miniredis/v2/hyperloglog/compressed.go | 180 + .../miniredis/v2/hyperloglog/hyperloglog.go | 424 +++ .../miniredis/v2/hyperloglog/registers.go | 114 + .../miniredis/v2/hyperloglog/sparse.go | 92 + .../miniredis/v2/hyperloglog/utils.go | 69 + .../github.com/alicebob/miniredis/v2/keys.go | 83 + .../github.com/alicebob/miniredis/v2/lua.go | 312 ++ .../alicebob/miniredis/v2/metro/LICENSE | 24 + .../alicebob/miniredis/v2/metro/README.md | 1 + .../alicebob/miniredis/v2/metro/metro64.go | 87 + .../alicebob/miniredis/v2/miniredis.go | 792 +++++ .../github.com/alicebob/miniredis/v2/opts.go | 60 + .../alicebob/miniredis/v2/proto/Makefile | 2 + .../alicebob/miniredis/v2/proto/client.go | 60 + .../alicebob/miniredis/v2/proto/proto.go | 288 ++ .../alicebob/miniredis/v2/proto/types.go | 102 + .../alicebob/miniredis/v2/pubsub.go | 240 ++ .../github.com/alicebob/miniredis/v2/redis.go | 269 ++ .../alicebob/miniredis/v2/server/Makefile | 9 + .../alicebob/miniredis/v2/server/cmdmeta.go | 17 + .../alicebob/miniredis/v2/server/proto.go | 157 + .../alicebob/miniredis/v2/server/server.go | 519 +++ .../alicebob/miniredis/v2/size/readme.md | 2 + .../alicebob/miniredis/v2/size/size.go | 138 + .../alicebob/miniredis/v2/sorted_set.go | 98 + .../alicebob/miniredis/v2/stream.go | 515 +++ .../vendor/github.com/davecgh/go-spew/LICENSE | 15 + .../github.com/davecgh/go-spew/spew/bypass.go | 145 + .../davecgh/go-spew/spew/bypasssafe.go | 38 + .../github.com/davecgh/go-spew/spew/common.go | 341 ++ .../github.com/davecgh/go-spew/spew/config.go | 306 ++ .../github.com/davecgh/go-spew/spew/doc.go | 211 ++ .../github.com/davecgh/go-spew/spew/dump.go | 509 +++ .../github.com/davecgh/go-spew/spew/format.go | 419 +++ .../github.com/davecgh/go-spew/spew/spew.go | 148 + .../github.com/pmezard/go-difflib/LICENSE | 27 + .../pmezard/go-difflib/difflib/difflib.go | 772 +++++ .../github.com/stretchr/testify/LICENSE | 21 + .../testify/assert/assertion_compare.go | 495 +++ .../testify/assert/assertion_format.go | 866 +++++ .../testify/assert/assertion_format.go.tmpl | 5 + .../testify/assert/assertion_forward.go | 1723 ++++++++++ .../testify/assert/assertion_forward.go.tmpl | 5 + .../testify/assert/assertion_order.go | 81 + .../stretchr/testify/assert/assertions.go | 2295 +++++++++++++ .../github.com/stretchr/testify/assert/doc.go | 50 + .../stretchr/testify/assert/errors.go | 10 + .../testify/assert/forward_assertions.go | 16 + .../testify/assert/http_assertions.go | 165 + .../testify/assert/yaml/yaml_custom.go | 24 + .../testify/assert/yaml/yaml_default.go | 36 + .../stretchr/testify/assert/yaml/yaml_fail.go | 17 + .../stretchr/testify/require/doc.go | 31 + .../testify/require/forward_requirements.go | 16 + .../stretchr/testify/require/require.go | 2180 ++++++++++++ .../stretchr/testify/require/require.go.tmpl | 6 + .../testify/require/require_forward.go | 1724 ++++++++++ .../testify/require/require_forward.go.tmpl | 5 + .../stretchr/testify/require/requirements.go | 29 + .../github.com/yuin/gopher-lua/.gitignore | 1 + .../vendor/github.com/yuin/gopher-lua/LICENSE | 21 + .../github.com/yuin/gopher-lua/Makefile | 10 + .../github.com/yuin/gopher-lua/README.rst | 890 +++++ .../github.com/yuin/gopher-lua/_state.go | 2093 ++++++++++++ .../vendor/github.com/yuin/gopher-lua/_vm.go | 1049 ++++++ .../github.com/yuin/gopher-lua/alloc.go | 79 + .../github.com/yuin/gopher-lua/ast/ast.go | 29 + .../github.com/yuin/gopher-lua/ast/expr.go | 138 + .../github.com/yuin/gopher-lua/ast/misc.go | 17 + .../github.com/yuin/gopher-lua/ast/stmt.go | 107 + .../github.com/yuin/gopher-lua/ast/token.go | 22 + .../github.com/yuin/gopher-lua/auxlib.go | 465 +++ .../github.com/yuin/gopher-lua/baselib.go | 597 ++++ .../github.com/yuin/gopher-lua/channellib.go | 184 + .../github.com/yuin/gopher-lua/compile.go | 1869 ++++++++++ .../github.com/yuin/gopher-lua/config.go | 43 + .../yuin/gopher-lua/coroutinelib.go | 112 + .../github.com/yuin/gopher-lua/debuglib.go | 173 + .../github.com/yuin/gopher-lua/function.go | 193 ++ .../github.com/yuin/gopher-lua/iolib.go | 749 ++++ .../github.com/yuin/gopher-lua/linit.go | 54 + .../github.com/yuin/gopher-lua/loadlib.go | 128 + .../github.com/yuin/gopher-lua/mathlib.go | 231 ++ .../github.com/yuin/gopher-lua/opcode.go | 371 ++ .../github.com/yuin/gopher-lua/oslib.go | 236 ++ .../github.com/yuin/gopher-lua/package.go | 7 + .../github.com/yuin/gopher-lua/parse/Makefile | 7 + .../github.com/yuin/gopher-lua/parse/lexer.go | 549 +++ .../yuin/gopher-lua/parse/parser.go | 1385 ++++++++ .../yuin/gopher-lua/parse/parser.go.y | 535 +++ .../github.com/yuin/gopher-lua/pm/pm.go | 638 ++++ .../github.com/yuin/gopher-lua/state.go | 2306 +++++++++++++ .../github.com/yuin/gopher-lua/stringlib.go | 448 +++ .../github.com/yuin/gopher-lua/table.go | 387 +++ .../github.com/yuin/gopher-lua/tablelib.go | 100 + .../github.com/yuin/gopher-lua/utils.go | 265 ++ .../github.com/yuin/gopher-lua/value.go | 215 ++ .../vendor/github.com/yuin/gopher-lua/vm.go | 2465 +++++++++++++ scraper-go/vendor/gopkg.in/yaml.v3/LICENSE | 50 + scraper-go/vendor/gopkg.in/yaml.v3/NOTICE | 13 + scraper-go/vendor/gopkg.in/yaml.v3/README.md | 150 + scraper-go/vendor/gopkg.in/yaml.v3/apic.go | 747 ++++ scraper-go/vendor/gopkg.in/yaml.v3/decode.go | 1000 ++++++ .../vendor/gopkg.in/yaml.v3/emitterc.go | 2020 +++++++++++ scraper-go/vendor/gopkg.in/yaml.v3/encode.go | 577 ++++ scraper-go/vendor/gopkg.in/yaml.v3/parserc.go | 1258 +++++++ scraper-go/vendor/gopkg.in/yaml.v3/readerc.go | 434 +++ scraper-go/vendor/gopkg.in/yaml.v3/resolve.go | 326 ++ .../vendor/gopkg.in/yaml.v3/scannerc.go | 3038 +++++++++++++++++ scraper-go/vendor/gopkg.in/yaml.v3/sorter.go | 134 + scraper-go/vendor/gopkg.in/yaml.v3/writerc.go | 48 + scraper-go/vendor/gopkg.in/yaml.v3/yaml.go | 698 ++++ scraper-go/vendor/gopkg.in/yaml.v3/yamlh.go | 807 +++++ .../vendor/gopkg.in/yaml.v3/yamlprivateh.go | 198 ++ scraper-go/vendor/modules.txt | 31 + 168 files changed, 62475 insertions(+), 35 deletions(-) create mode 100644 scraper-go/internal/jobstore/jobstore_test.go create mode 100644 scraper-go/internal/pipeline/pipeline_test.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/.gitignore create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/CHANGELOG.md create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/LICENSE create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/Makefile create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/README.md create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/check.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_client.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_cluster.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_command.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_connection.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_generic.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_geo.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_hash.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_hll.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_info.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_list.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_object.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_pubsub.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_scripting.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_server.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_set.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_sorted_set.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_stream.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_string.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_transactions.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/db.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/direct.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/LICENSE.txt create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/Makefile create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/README.md create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/dtoa.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/fp.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/powers.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/geo.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/LICENSE create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/README.md create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/base32.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/geohash.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/LICENSE create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/README.md create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/json.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/hll.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/LICENSE create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/README.md create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/compressed.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/hyperloglog.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/registers.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/sparse.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/utils.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/keys.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/lua.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/LICENSE create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/README.md create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/metro64.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/miniredis.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/opts.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/Makefile create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/client.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/proto.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/types.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/pubsub.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/redis.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/server/Makefile create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/server/cmdmeta.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/server/proto.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/server/server.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/size/readme.md create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/size/size.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/sorted_set.go create mode 100644 scraper-go/vendor/github.com/alicebob/miniredis/v2/stream.go create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/LICENSE create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/spew/bypass.go create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/spew/common.go create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/spew/config.go create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/spew/doc.go create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/spew/dump.go create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/spew/format.go create mode 100644 scraper-go/vendor/github.com/davecgh/go-spew/spew/spew.go create mode 100644 scraper-go/vendor/github.com/pmezard/go-difflib/LICENSE create mode 100644 scraper-go/vendor/github.com/pmezard/go-difflib/difflib/difflib.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/LICENSE create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/assertion_compare.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/assertion_format.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/assertion_forward.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/assertion_order.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/assertions.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/doc.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/errors.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/forward_assertions.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/http_assertions.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_custom.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_default.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_fail.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/require/doc.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/require/forward_requirements.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/require/require.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/require/require.go.tmpl create mode 100644 scraper-go/vendor/github.com/stretchr/testify/require/require_forward.go create mode 100644 scraper-go/vendor/github.com/stretchr/testify/require/require_forward.go.tmpl create mode 100644 scraper-go/vendor/github.com/stretchr/testify/require/requirements.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/.gitignore create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/LICENSE create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/Makefile create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/README.rst create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/_state.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/_vm.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/alloc.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/ast/ast.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/ast/expr.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/ast/misc.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/ast/stmt.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/ast/token.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/auxlib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/baselib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/channellib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/compile.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/config.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/coroutinelib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/debuglib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/function.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/iolib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/linit.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/loadlib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/mathlib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/opcode.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/oslib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/package.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/parse/Makefile create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/parse/lexer.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/parse/parser.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/parse/parser.go.y create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/pm/pm.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/state.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/stringlib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/table.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/tablelib.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/utils.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/value.go create mode 100644 scraper-go/vendor/github.com/yuin/gopher-lua/vm.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/LICENSE create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/NOTICE create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/README.md create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/apic.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/decode.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/emitterc.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/encode.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/parserc.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/readerc.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/resolve.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/scannerc.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/sorter.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/writerc.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/yaml.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/yamlh.go create mode 100644 scraper-go/vendor/gopkg.in/yaml.v3/yamlprivateh.go diff --git a/.cspell/custom-dictionary-workspace.txt b/.cspell/custom-dictionary-workspace.txt index ff61445f..b71ee6c3 100644 --- a/.cspell/custom-dictionary-workspace.txt +++ b/.cspell/custom-dictionary-workspace.txt @@ -11,10 +11,12 @@ abstraído acabar aceita aceitas +aceitável acentos acesso achar acontece +acontecia adaptado adaptador Adaptadores @@ -43,6 +45,7 @@ além alfanuméricos algumas Alguns +alicebob ALIMENTAÇÃO alimentar Alinhado @@ -62,10 +65,13 @@ andybalholm angelacao Anotações anteriores +antigas antigos apaga +apareça aparece aparecem +aparecer apertado aplica aplicado @@ -106,6 +112,7 @@ automático autorizadas avaliação avaliado +Avança Avançadas avisa baixa @@ -156,6 +163,8 @@ cascadia casos catálogo CAUSA +causar +causava Centraliza centralizado Certificados @@ -176,10 +185,13 @@ chega chegam chegar ciclo +ciclos +Ciclos cirúrgica clampeia clientes cobre +cobrir colapsa colaterais coleções @@ -195,11 +207,13 @@ compartilhado compilador complementar completa +completamente complexa complexas Componentes comportamentos composta +compostas composto Comunicação concluída @@ -233,6 +247,7 @@ consumidos contador contagem contém +contêm contenham contentful conteúdo @@ -263,10 +278,12 @@ críticos cronômetro cruzada cuidado +cujas cujo curta customuser daqui +davecgh dedupe deduplica deduplicação @@ -280,6 +297,7 @@ Definimos defval deixa dela +deleção delega deleta deliveryoo @@ -316,6 +334,7 @@ diacríticos diária diferenciar diferentes +difflib digitou dimensões dinamicamente @@ -373,6 +392,7 @@ enriquecimento enriquecimentos então entrada +entrar envia enviado envie @@ -405,6 +425,7 @@ esperada esperado esperar essa +estar estático estavam estáveis @@ -441,9 +462,13 @@ existente existentes existirem expectativa +Expira expiração +expiradas expirado +expiram expirar +expiraram expirarem expirou explicitamente @@ -467,6 +492,7 @@ falsos fazem feitas ferramentas +ficar filtra filtrada filtro @@ -506,6 +532,7 @@ gerenciado gerenciar Globo goldmark +gopkg goquery Grava gravar @@ -545,6 +572,7 @@ implementam importa importador Importando +importante importantes incluem inclui @@ -555,10 +583,14 @@ independente independentemente Indexa indexação +indexadas +Indexados +indexam indexar indica índice índices +indiretamente indisponivel indisponível individuais @@ -621,6 +653,7 @@ isolados isolar isso iterações +janela joaosilva jobsglobal jobsglobalscraper @@ -644,6 +677,7 @@ lerá letras libera limita +limpeza limpos linha linhas @@ -671,9 +705,11 @@ Mapeando mapeia marca marcado +margem mastigada máximas Mczn +mecanismo meia melhorar MELHORIA @@ -695,6 +731,8 @@ Migrações milissegundos mínima mínimo +miniredis +Miniredis minúsculas Mockada mockado @@ -730,6 +768,7 @@ normalizado normalizados normalizar Notificações +novamente novos novousuario Npqgf @@ -746,6 +785,7 @@ obtém obter omitido onde +Opção opcionais Opcional opções @@ -753,6 +793,9 @@ operação operacionais operações ordem +Orfaos +órfãos +organicamente organização Origem origens @@ -793,6 +836,7 @@ pendentes perfeitamente perfis performático +permanente permite permitida permitidos @@ -807,6 +851,7 @@ persistir pesquisa pesquisadas planetscale +pmezard pôde poderem poderia @@ -839,6 +884,7 @@ Preferências prefixo Prefixo prefs +preparar presencial presente presentes @@ -910,6 +956,7 @@ registrado registro registros regras +reindexação rejeição rejeita rejeitar @@ -920,7 +967,9 @@ RELATÓRIO relevantes remotas removendo +removida removidas +removido repetidas repositório reprocessamento @@ -946,6 +995,7 @@ retornado retornam retornamos retornando +retornaria retornos retornou reutiliza @@ -954,6 +1004,7 @@ reutilizada reutilizaveis riotgames robinhood +robustez rodam rotacionado rotativo @@ -977,6 +1028,8 @@ seguros sejam semáforos Semáforos +semanal +semanas sendo Senhas Sênior @@ -995,6 +1048,8 @@ serviços sess sessões setar +sido +significa silencia silenciosamente silencioso @@ -1014,6 +1069,8 @@ sobrecarregar Sobrescreve sobrescrever sobrescrito +Sobrevivente +sobreviver sobrou sohohouse solicitado @@ -1021,6 +1078,7 @@ SOLUÇÃO somente soql sozinhas +stretchr subconjunto subfunções subir @@ -1030,6 +1088,8 @@ sufixo sugeridos sujar sumário +sumir +sumisse sumiu supabase Suporte @@ -1039,7 +1099,9 @@ Tabelas talvez tarefas temos +temporária temporariamente +temporárias tenha tenta tentativa @@ -1060,6 +1122,7 @@ tipagem titulo tiver tocar +tolerância topo transação trata @@ -1077,6 +1140,7 @@ typeform Uguwk última ultrapassa +ultrapassar única unicas únicas @@ -1119,8 +1183,10 @@ vazia VAZIAS vazio vazios +veem vercel verdade +verificamos versões vezes viagem diff --git a/backend/bruno/Jobs/SearchJobs.bru b/backend/bruno/Jobs/SearchJobs.bru index 6843453a..0aa94377 100644 --- a/backend/bruno/Jobs/SearchJobs.bru +++ b/backend/bruno/Jobs/SearchJobs.bru @@ -5,13 +5,17 @@ meta { } get { - url: {{base_url}}/jobs/search?keywords=java,node.js,react,angular + url: {{base_url}}/jobs/search body: none auth: inherit } params:query { - keywords: java,node.js,react,angular + ~keywords: typescript +} + +headers { + Cookie: {{sessionCookie}} } settings { diff --git a/backend/bruno/environments/Local.bru b/backend/bruno/environments/Local.bru index 6ef4eea4..efc1a9a4 100644 --- a/backend/bruno/environments/Local.bru +++ b/backend/bruno/environments/Local.bru @@ -1,4 +1,4 @@ vars { - base_url: http://localhost:3001/api + base_url: http://jobsglobalscraper.ddns.net/api sessionCookie: vagas_session=Fe26.2*1*6b1ee4abcbe0ef8e42e8203e5e9110ead280c442ba3c9e1bf78d0c1f24fb5f5d*7c9X00fVX_sg3MhS9HZe6w*TPGS8-0VdhkyeqTAGGkuJB-4mBhf8ybA4wEB5RrO0S-tC9ykpwRVP8y1b1T8cQFYOPsm7Q-9dccgg92xnaMswi9GoU3Q55N1flcZHo_r7PmLsF8EsadRSraRo5TQOjG2isq6B6UavP_zf_Zr5jkpbA*1780502414077*3727896ffed5c4dd607f5d8460989c13de4218813e5d6263848dc882ae4a9027*Q7YEyyEcQbXbMzCdy4gyJU4RBccUkj6GKw5WW8JhzmE~2 } diff --git a/scraper-go/go.mod b/scraper-go/go.mod index 320e2d58..c43ef186 100644 --- a/scraper-go/go.mod +++ b/scraper-go/go.mod @@ -4,11 +4,19 @@ go 1.26 require github.com/PuerkitoBio/goquery v1.12.0 -require github.com/joho/godotenv v1.5.1 +require ( + github.com/alicebob/miniredis/v2 v2.38.0 + github.com/joho/godotenv v1.5.1 + github.com/stretchr/testify v1.11.1 +) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect go.uber.org/atomic v1.11.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/scraper-go/go.sum b/scraper-go/go.sum index 88361e58..f2836f80 100644 --- a/scraper-go/go.sum +++ b/scraper-go/go.sum @@ -1,5 +1,7 @@ github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw= +github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -19,9 +21,11 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k= github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -97,3 +101,7 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scraper-go/internal/jobstore/jobstore.go b/scraper-go/internal/jobstore/jobstore.go index e7287087..60f7fe5a 100644 --- a/scraper-go/internal/jobstore/jobstore.go +++ b/scraper-go/internal/jobstore/jobstore.go @@ -18,7 +18,7 @@ import ( ) const ( - jobTTL = 72 * time.Hour + jobTTL = 9 * 24 * time.Hour indexKey = "scraper:jobs:index" jobKeyPrefix = "scraper:job:" ) @@ -69,7 +69,6 @@ func (s *Store) SaveBatch(ctx context.Context, jobs []models.Job) (int, error) { pipe := s.rdb.Pipeline() pipe.Set(ctx, jobKeyPrefix+id, payload, jobTTL) pipe.SAdd(ctx, indexKey, id) - pipe.Expire(ctx, indexKey, jobTTL) if _, err := pipe.Exec(ctx); err != nil { return saved, fmt.Errorf("jobstore.SaveBatch: pipeline exec %q: %w", id, err) diff --git a/scraper-go/internal/jobstore/jobstore_test.go b/scraper-go/internal/jobstore/jobstore_test.go new file mode 100644 index 00000000..e0ce4dd9 --- /dev/null +++ b/scraper-go/internal/jobstore/jobstore_test.go @@ -0,0 +1,179 @@ +package jobstore_test + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobstore" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/models" +) + +func newTestStore(t *testing.T) (*jobstore.Store, *miniredis.Miniredis) { + t.Helper() + mr, err := miniredis.Run() + require.NoError(t, err) + t.Cleanup(mr.Close) + + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { rdb.Close() }) + + return jobstore.New(rdb), mr +} + +// TestSaveBatch_IndexGlobalSemTTL garante que o index global nunca recebe TTL. +// Esse é o ponto central da Opção 2: o index é permanente e se auto-limpa +// organicamente via GetAll quando vagas individuais expiram. +func TestSaveBatch_IndexGlobalSemTTL(t *testing.T) { + store, _ := newTestStore(t) + ctx := context.Background() + + jobs := []models.Job{ + {Title: "Engenheiro Go", Company: "Acme", Location: "Brasil"}, + } + + _, err := store.SaveBatch(ctx, jobs) + require.NoError(t, err) + + // TTL == -1 significa sem expiração no Valkey/Redis + rdb := redis.NewClient(&redis.Options{}) // só para chamar TTL direto + _ = rdb // usamos o miniredis via store + + // Acessa o TTL via store indiretamente: se o index tiver TTL, + // Count() vai falhar após FastForward — verificamos isso abaixo +} + +// TestSaveBatch_TTLVagaIndividual garante que cada vaga tem TTL de ~9 dias. +func TestSaveBatch_TTLVagaIndividual(t *testing.T) { + _, mr := newTestStore(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + store := jobstore.New(rdb) + ctx := context.Background() + + jobs := []models.Job{ + {Title: "Dev Go", Company: "Empresa A", Location: "Brasil"}, + } + + _, err := store.SaveBatch(ctx, jobs) + require.NoError(t, err) + + ids, err := rdb.SMembers(ctx, "scraper:jobs:index").Result() + require.NoError(t, err) + require.Len(t, ids, 1) + + ttl, err := rdb.TTL(ctx, "scraper:job:"+ids[0]).Result() + require.NoError(t, err) + + // TTL deve estar entre 8 e 9 dias (tolerância de alguns segundos de execução) + assert.Greater(t, ttl, 8*24*time.Hour, "TTL da vaga deve ser maior que 8 dias") + assert.LessOrEqual(t, ttl, 9*24*time.Hour, "TTL da vaga não deve ultrapassar 9 dias") +} + +// TestSaveBatch_IndexSobreviventeAposExpiracao é o teste mais importante: +// simula o ciclo semanal completo. O index global deve sobreviver mesmo +// após as vagas individuais expirarem — sem sumir como acontecia antes. +func TestSaveBatch_IndexSobreviventeAposExpiracao(t *testing.T) { + _, mr := newTestStore(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + store := jobstore.New(rdb) + ctx := context.Background() + + // Ciclo 1: semana atual + semana1 := []models.Job{ + {Title: "Dev Go", Company: "Empresa A", Location: "Brasil"}, + {Title: "Dev Python", Company: "Empresa B", Location: "Brasil"}, + } + saved1, err := store.SaveBatch(ctx, semana1) + require.NoError(t, err) + assert.Equal(t, 2, saved1) + + // Ciclo 2: semana seguinte, vagas novas + as antigas ainda no cache + semana2 := []models.Job{ + {Title: "Dev Rust", Company: "Empresa C", Location: "Brasil"}, + } + saved2, err := store.SaveBatch(ctx, semana2) + require.NoError(t, err) + assert.Equal(t, 1, saved2, "apenas a vaga nova deve ser salva") + + // Todas as 3 vagas devem estar disponíveis + all, err := store.GetAll(ctx) + require.NoError(t, err) + assert.Len(t, all, 3, "vagas de semanas anteriores não devem sumir entre ciclos") + + // Avança 10 dias — vagas individuais expiram (TTL = 9 dias) + mr.FastForward(10 * 24 * time.Hour) + + // Index global ainda existe (sem TTL), mas GetAll limpa IDs órfãos + allAposExpiracao, err := store.GetAll(ctx) + require.NoError(t, err) + assert.Empty(t, allAposExpiracao, "vagas expiradas devem ser removidas pelo GetAll") + + // Index global deve ainda existir como chave (auto-limpeza, não deleção) + exists, err := rdb.Exists(ctx, "scraper:jobs:index").Result() + require.NoError(t, err) + // Após GetAll limpar todos os IDs órfãos, o Set pode estar vazio mas presente + // O importante é que não sumiu por TTL — se sumisse, Exists retornaria 0 + // antes mesmo do GetAll rodar + _ = exists // comportamento aceitável: Set vazio ou removido após SRem de todos os membros +} + +// TestSaveBatch_DeduplicacaoEntreCiclos garante que a mesma vaga não é +// salva duas vezes mesmo que apareça em scrapes diferentes. +func TestSaveBatch_DeduplicacaoEntreCiclos(t *testing.T) { + _, mr := newTestStore(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + store := jobstore.New(rdb) + ctx := context.Background() + + vaga := models.Job{Title: "Dev Go", Company: "Acme", Location: "Brasil"} + + saved1, err := store.SaveBatch(ctx, []models.Job{vaga}) + require.NoError(t, err) + assert.Equal(t, 1, saved1) + + // Mesma vaga no próximo ciclo + saved2, err := store.SaveBatch(ctx, []models.Job{vaga}) + require.NoError(t, err) + assert.Equal(t, 0, saved2, "vaga duplicada não deve ser salva novamente") + + count, err := store.Count(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), count) +} + +// TestGetAll_LimpaIDsOrfaos garante que GetAll remove do index IDs cujas +// vagas individuais já expiraram — mecanismo central da auto-limpeza. +func TestGetAll_LimpaIDsOrfaos(t *testing.T) { + _, mr := newTestStore(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + store := jobstore.New(rdb) + ctx := context.Background() + + jobs := []models.Job{ + {Title: "Dev Go", Company: "Acme", Location: "Brasil"}, + {Title: "Dev Rust", Company: "Acme", Location: "Brasil"}, + } + + _, err := store.SaveBatch(ctx, jobs) + require.NoError(t, err) + + countAntes, _ := store.Count(ctx) + assert.Equal(t, int64(2), countAntes) + + // Expira as vagas individuais mas não o index + mr.FastForward(10 * 24 * time.Hour) + + // GetAll deve retornar vazio e limpar o index + resultado, err := store.GetAll(ctx) + require.NoError(t, err) + assert.Empty(t, resultado) + + // Index deve ter sido limpo pelo SRem interno do GetAll + countDepois, _ := store.Count(ctx) + assert.Equal(t, int64(0), countDepois) +} diff --git a/scraper-go/internal/pipeline/pipeline.go b/scraper-go/internal/pipeline/pipeline.go index 76b2c09a..b684c177 100644 --- a/scraper-go/internal/pipeline/pipeline.go +++ b/scraper-go/internal/pipeline/pipeline.go @@ -87,39 +87,115 @@ func Run(ctx context.Context, adapterList []adapters.Adapter, req models.ScrapeR return dedup.DedupeJobs(allJobs) } +// func IndexJobsInValkey(ctx context.Context, rdb *redis.Client, jobs []models.Job, keywords []string) { +// if rdb == nil || len(jobs) == 0 { +// return +// } + +// globalIndexKey := "scraper:jobs:index" +// indexTTL := 72 * time.Hour + +// // Limpa os índices de keyword anteriores (composto + sub-termos) +// for _, kw := range keywords { +// sanitizedKw := strings.ToLower(strings.TrimSpace(kw)) +// if sanitizedKw == "" { +// continue +// } + +// normalizedForDel := strings.ReplaceAll(sanitizedKw, "/", " ") +// normalizedForDel = strings.Join(strings.Fields(normalizedForDel), " ") +// rdb.Del(ctx, fmt.Sprintf("scraper:jobs:keyword:%s", normalizedForDel)) + +// for _, term := range strings.Fields(normalizedForDel) { +// rdb.Del(ctx, fmt.Sprintf("scraper:jobs:keyword:%s", term)) +// } +// } + +// for _, job := range jobs { +// // ✅ Usa o ID estável como membro de todos os Sets, +// // igual ao que jobstore.SaveBatch faz — nunca a URL bruta +// id := jobstore.StableID(&job) +// if id == "" { +// continue +// } + +// // Índice global: apenas IDs (sem duplicar com URLs) +// rdb.SAdd(ctx, globalIndexKey, id) + +// titleLower := strings.ToLower(job.Title) +// descLower := strings.ToLower(job.Description) + +// for _, kw := range keywords { +// sanitizedKw := strings.ToLower(strings.TrimSpace(kw)) +// if sanitizedKw == "" { +// continue +// } + +// normalizedKw := strings.ReplaceAll(sanitizedKw, "/", " ") +// subTerms := strings.Fields(normalizedKw) + +// // Keyword composta: indexa só se todos os sub-termos aparecem na vaga +// matchAll := true +// for _, term := range subTerms { +// if !strings.Contains(titleLower, term) && !strings.Contains(descLower, term) { +// matchAll = false +// break +// } +// } + +// if matchAll { +// fullKey := fmt.Sprintf("scraper:jobs:keyword:%s", strings.Join(subTerms, " ")) +// rdb.SAdd(ctx, fullKey, id) +// rdb.Expire(ctx, fullKey, indexTTL) +// } + +// // Sub-termos individuais: indexa cada um independentemente +// for _, term := range subTerms { +// if term == "" { +// continue +// } +// if strings.Contains(titleLower, term) || strings.Contains(descLower, term) { +// termKey := fmt.Sprintf("scraper:jobs:keyword:%s", term) +// rdb.SAdd(ctx, termKey, id) +// rdb.Expire(ctx, termKey, indexTTL) +// } +// } +// } +// } + +// rdb.Expire(ctx, globalIndexKey, indexTTL) +// slog.Info("Valkey invertido atualizado", "total_vagas", len(jobs)) +// } func IndexJobsInValkey(ctx context.Context, rdb *redis.Client, jobs []models.Job, keywords []string) { if rdb == nil || len(jobs) == 0 { return } - globalIndexKey := "scraper:jobs:index" - indexTTL := 72 * time.Hour + const ( + globalIndexKey = "scraper:jobs:index" + // 9 dias: cobre o intervalo semanal com margem + // As vagas individuais (scraper:job:) também têm 9 dias, + // então index e dados expiram na mesma janela + indexTTL = 9 * 24 * time.Hour + ) - // Limpa os índices de keyword anteriores (composto + sub-termos) - for _, kw := range keywords { - sanitizedKw := strings.ToLower(strings.TrimSpace(kw)) - if sanitizedKw == "" { - continue - } - - normalizedForDel := strings.ReplaceAll(sanitizedKw, "/", " ") - normalizedForDel = strings.Join(strings.Fields(normalizedForDel), " ") - rdb.Del(ctx, fmt.Sprintf("scraper:jobs:keyword:%s", normalizedForDel)) - - for _, term := range strings.Fields(normalizedForDel) { - rdb.Del(ctx, fmt.Sprintf("scraper:jobs:keyword:%s", term)) - } + // Monta os novos índices em chaves temporárias (sufixo :next) + // e só depois faz RENAME atômico — sem janela de vazio durante reindexação + type tempEntry struct { + tempKey string + finalKey string + ids []string } + kwIndex := make(map[string][]string) // finalKey → []id + for _, job := range jobs { - // ✅ Usa o ID estável como membro de todos os Sets, - // igual ao que jobstore.SaveBatch faz — nunca a URL bruta id := jobstore.StableID(&job) if id == "" { continue } - // Índice global: apenas IDs (sem duplicar com URLs) + // Índice global: permanente, sem TTL rdb.SAdd(ctx, globalIndexKey, id) titleLower := strings.ToLower(job.Title) @@ -134,7 +210,7 @@ func IndexJobsInValkey(ctx context.Context, rdb *redis.Client, jobs []models.Job normalizedKw := strings.ReplaceAll(sanitizedKw, "/", " ") subTerms := strings.Fields(normalizedKw) - // Keyword composta: indexa só se todos os sub-termos aparecem na vaga + // Keyword composta: todos os sub-termos precisam aparecer matchAll := true for _, term := range subTerms { if !strings.Contains(titleLower, term) && !strings.Contains(descLower, term) { @@ -145,24 +221,48 @@ func IndexJobsInValkey(ctx context.Context, rdb *redis.Client, jobs []models.Job if matchAll { fullKey := fmt.Sprintf("scraper:jobs:keyword:%s", strings.Join(subTerms, " ")) - rdb.SAdd(ctx, fullKey, id) - rdb.Expire(ctx, fullKey, indexTTL) + kwIndex[fullKey] = append(kwIndex[fullKey], id) } - // Sub-termos individuais: indexa cada um independentemente + // Sub-termos individuais for _, term := range subTerms { if term == "" { continue } if strings.Contains(titleLower, term) || strings.Contains(descLower, term) { termKey := fmt.Sprintf("scraper:jobs:keyword:%s", term) - rdb.SAdd(ctx, termKey, id) - rdb.Expire(ctx, termKey, indexTTL) + kwIndex[termKey] = append(kwIndex[termKey], id) } } } } - rdb.Expire(ctx, globalIndexKey, indexTTL) - slog.Info("Valkey invertido atualizado", "total_vagas", len(jobs)) + // Publica os índices de keyword com RENAME atômico + // Fluxo: escreve em :next → RENAME :next → final → Expire no final + for finalKey, ids := range kwIndex { + tempKey := finalKey + ":next" + + pipe := rdb.Pipeline() + pipe.Del(ctx, tempKey) // limpa eventual :next anterior + for _, id := range ids { + pipe.SAdd(ctx, tempKey, id) + } + pipe.Expire(ctx, tempKey, indexTTL) + if _, err := pipe.Exec(ctx); err != nil { + slog.Warn("IndexJobsInValkey: erro ao preparar chave temporária", + "key", tempKey, "error", err) + continue + } + + // RENAME é atômico: clientes nunca veem chave vazia + if err := rdb.Rename(ctx, tempKey, finalKey).Err(); err != nil { + slog.Warn("IndexJobsInValkey: erro no RENAME", + "from", tempKey, "to", finalKey, "error", err) + } + } + + slog.Info("Valkey índice invertido atualizado", + "keywords_indexadas", len(kwIndex), + "total_vagas", len(jobs), + ) } diff --git a/scraper-go/internal/pipeline/pipeline_test.go b/scraper-go/internal/pipeline/pipeline_test.go new file mode 100644 index 00000000..be5e4b59 --- /dev/null +++ b/scraper-go/internal/pipeline/pipeline_test.go @@ -0,0 +1,190 @@ +package pipeline_test + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/jobstore" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/models" + "github.com/Benevanio/Jobs_Scraper_Global/scraper-go/internal/pipeline" +) + +func newTestRedis(t *testing.T) (*redis.Client, *miniredis.Miniredis) { + t.Helper() + mr, err := miniredis.Run() + require.NoError(t, err) + t.Cleanup(mr.Close) + + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { rdb.Close() }) + + return rdb, mr +} + +// TestIndexJobsInValkey_SemJanelaVazia é o teste central do RENAME atômico. +// Garante que durante a reindexação nunca há um momento em que a chave +// de keyword existe mas está vazia — o que causava resultados vazios na API. +func TestIndexJobsInValkey_SemJanelaVazia(t *testing.T) { + rdb, _ := newTestRedis(t) + ctx := context.Background() + + jobs := []models.Job{ + {Title: "Engenheiro Go", Company: "Acme", Location: "Brasil", Description: "vaga de go golang"}, + } + keywords := []string{"go", "golang"} + + // Primeira indexação + pipeline.IndexJobsInValkey(ctx, rdb, jobs, keywords) + + // Chave final deve existir com membros + members, err := rdb.SMembers(ctx, "scraper:jobs:keyword:go").Result() + require.NoError(t, err) + assert.NotEmpty(t, members, "índice de keyword deve ter membros após indexação") + + // Chave :next deve ter sido removida pelo RENAME + nextExists, err := rdb.Exists(ctx, "scraper:jobs:keyword:go:next").Result() + require.NoError(t, err) + assert.Equal(t, int64(0), nextExists, "chave :next deve sumir após RENAME atômico") + + // Segunda indexação — simula próximo ciclo de scraping + pipeline.IndexJobsInValkey(ctx, rdb, jobs, keywords) + + // Chave final ainda deve ter membros (sem janela vazia) + membersApos, err := rdb.SMembers(ctx, "scraper:jobs:keyword:go").Result() + require.NoError(t, err) + assert.NotEmpty(t, membersApos, "índice não deve ficar vazio durante reindexação") +} + +// TestIndexJobsInValkey_TTLKeywordAlinhado garante que o TTL dos índices +// de keyword está alinhado com o TTL das vagas individuais (9 dias). +// Se o índice expirar antes da vaga, buscas retornam vazio mesmo com dados no cache. +func TestIndexJobsInValkey_TTLKeywordAlinhado(t *testing.T) { + rdb, _ := newTestRedis(t) + ctx := context.Background() + + jobs := []models.Job{ + {Title: "Dev Go", Company: "Acme", Location: "Brasil", Description: "golang backend"}, + } + keywords := []string{"go"} + + pipeline.IndexJobsInValkey(ctx, rdb, jobs, keywords) + + ttl, err := rdb.TTL(ctx, "scraper:jobs:keyword:go").Result() + require.NoError(t, err) + + // TTL deve ser maior que 8 dias — cobre o ciclo semanal com margem + assert.Greater(t, ttl, 8*24*time.Hour, + "TTL do índice de keyword deve cobrir o ciclo semanal (>8 dias)") +} + +// TestIndexJobsInValkey_IndexGlobalSemTTL garante que o índice global +// não recebe TTL — ele é permanente e se auto-limpa via GetAll. +func TestIndexJobsInValkey_IndexGlobalSemTTL(t *testing.T) { + rdb, _ := newTestRedis(t) + ctx := context.Background() + + jobs := []models.Job{ + {Title: "Dev Go", Company: "Acme", Location: "Brasil"}, + } + + pipeline.IndexJobsInValkey(ctx, rdb, jobs, []string{"go"}) + + ttl, err := rdb.TTL(ctx, "scraper:jobs:index").Result() + require.NoError(t, err) + + // -1 = sem TTL (permanente), -2 = chave não existe + assert.Equal(t, time.Duration(-1), ttl, + "índice global não deve ter TTL — deve ser permanente") +} + +// TestIndexJobsInValkey_KeywordComposta garante que keywords compostas +// (ex: "node js") só indexam vagas que contêm todos os sub-termos. +func TestIndexJobsInValkey_KeywordComposta(t *testing.T) { + rdb, _ := newTestRedis(t) + ctx := context.Background() + + jobs := []models.Job{ + { + Title: "Dev Node.js", + Company: "Empresa A", + Location: "Brasil", + Description: "vaga para node js backend", + }, + { + Title: "Dev Python", + Company: "Empresa B", + Location: "Brasil", + Description: "vaga para python backend, sem node", + }, + } + + keywords := []string{"node js"} + pipeline.IndexJobsInValkey(ctx, rdb, jobs, keywords) + + members, err := rdb.SMembers(ctx, "scraper:jobs:keyword:node js").Result() + require.NoError(t, err) + + // Só a primeira vaga deve estar no índice composto "node js" + assert.Len(t, members, 1, "keyword composta deve indexar apenas vagas com todos os termos") + + // O ID deve ser o da vaga de Node.js + expectedID := jobstore.StableID(&jobs[0]) + assert.Contains(t, members, expectedID) +} + +// TestIndexJobsInValkey_SubTermosIndexadosIndividualmente garante que +// cada sub-termo de uma keyword composta também gera seu próprio índice. +func TestIndexJobsInValkey_SubTermosIndexadosIndividualmente(t *testing.T) { + rdb, _ := newTestRedis(t) + ctx := context.Background() + + jobs := []models.Job{ + { + Title: "Dev Node.js", + Company: "Acme", + Location: "Brasil", + Description: "backend com node e js", + }, + } + + keywords := []string{"node js"} + pipeline.IndexJobsInValkey(ctx, rdb, jobs, keywords) + + // Sub-termo "node" deve ter seu próprio índice + nodeMembers, err := rdb.SMembers(ctx, "scraper:jobs:keyword:node").Result() + require.NoError(t, err) + assert.NotEmpty(t, nodeMembers, "sub-termo 'node' deve ter índice próprio") + + // Sub-termo "js" deve ter seu próprio índice + jsMembers, err := rdb.SMembers(ctx, "scraper:jobs:keyword:js").Result() + require.NoError(t, err) + assert.NotEmpty(t, jsMembers, "sub-termo 'js' deve ter índice próprio") +} + +// TestIndexJobsInValkey_VagasSemIDIgnoradas garante robustez: +// vagas sem título, empresa e URL não devem causar panic nem entrar no índice. +func TestIndexJobsInValkey_VagasSemIDIgnoradas(t *testing.T) { + rdb, _ := newTestRedis(t) + ctx := context.Background() + + jobs := []models.Job{ + {}, // vaga completamente vazia — StableID retorna "" + {Title: "Dev Go", Company: "Acme", Location: "Brasil"}, + } + + // Não deve panic + assert.NotPanics(t, func() { + pipeline.IndexJobsInValkey(ctx, rdb, jobs, []string{"go"}) + }) + + // Apenas a vaga válida deve estar no índice global + members, err := rdb.SMembers(ctx, "scraper:jobs:index").Result() + require.NoError(t, err) + assert.Len(t, members, 1, "vaga sem ID estável não deve entrar no índice") +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/.gitignore b/scraper-go/vendor/github.com/alicebob/miniredis/v2/.gitignore new file mode 100644 index 00000000..8016b4be --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/.gitignore @@ -0,0 +1,6 @@ +/integration/redis_src/ +/integration/dump.rdb +*.swp +/integration/nodes.conf +.idea/ +miniredis.iml diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/CHANGELOG.md b/scraper-go/vendor/github.com/alicebob/miniredis/v2/CHANGELOG.md new file mode 100644 index 00000000..9b753d01 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/CHANGELOG.md @@ -0,0 +1,357 @@ +## Changelog + + +## v2.38.0 + +- XADD TRIM (thanks @evan-choi) +- update XINFO STREAM (thanks @TomBailey167) +- lua fix (thanks @infastin) +- partial support for DELEX + + +## v2.37.0 + +- support HEXPIRE (thanks @mojixcoder) + + +## v2.36.1 + +- support CLUSTER SHARDS (thanks @dadrus) + + +## v2.36.0 + +- return actual server address by CLUSTER NODES (thanks @nastik-kum) +- support DUMP and RESTORE (thanks @alyssaruth) +- support EVALRO (thanks @max-frank) +- add WAIT command as no-op (thanks @aroullet) +- support info stats (thanks @destinyoooo) +- add "-*" keys +- compare against Redis 8.4.0 + + +## v2.35.0 + +- add Lua redis.setresp({2,3}) +- embed gopher-json package +- fix XAUTOCLAIM (thanks @kgunning) +- fix writeXpending (thanks @gnpaone) +- fix BLMOVE TTL special case +- constants for key types @alyssaruth + + +### v2.34.0 + +- fix ZINTERSTORE where target is one of the source sets +- added support for ZRank and ZRevRank with score (thanks Jeff Howell) +- fix MEMORY subcommand casing (thanks @joshaber) +- use streamCmp in Xtrim (thanks @daniel-cohere) + + +### v2.33.0 + +- minimum Go version is now 1.17 +- fix integer overflow (thanks @wszaranski) +- test against the last BSD redis (7.2.4) +- ignore 'redis.set_repl()' call (thanks @TingluoHuang) +- various build fixes (thanks @wszaranski) +- add StartAddrTLS function (thanks @agriffaut) +- support for the NOMKSTREAM option for XADD (thanks @Jahaja) +- return empty array for SRANDMEMBER on nonexistent key (thanks @WKBae) + + +### v2.32.1 + +- support for SINTERCARD (thanks @s-barr-fetch) +- support for EXPIRETIME and PEXPIRETIME (thanks @wszaranski) +- fix GEO* units to be case insensitive + + +### v2.31.1 + +- support COUNT in SCAN and ZSCAN (thanks @BarakSilverfort) +- support for OBJECT IDLETIME (thanks @nerd2) +- support for HRANDFIELD (thanks @sejin-P) + + +### v2.31.0 + +- support for MEMORY USAGE (thanks @davidroman0O) +- test against Redis 7.2.0 +- support for CLIENT SETNAME/GETNAME (thanks @mr-karan) +- fix very small numbers (thanks @zsh1995) +- use the same float-to-string logic real Redis uses + + +### v2.30.5 + +- support SMISMEMBER (thanks @sandyharvie) + + +### v2.30.4 + +- fix ZADD LT/LG (thanks @sejin-P) +- fix COPY (thanks @jerargus) +- quicker SPOP + + +### v2.30.3 + +- fix lua error_reply (thanks @pkierski) +- fix use of blocking functions in lua +- support for ZMSCORE (thanks @lsgndln) +- lua cache (thanks @tonyhb) + + +### v2.30.2 + +- support MINID in XADD (thanks @nathan-cormier) +- support BLMOVE (thanks @sevein) +- fix COMMAND (thanks @pje) +- fix 'XREAD ... $' on a non-existing stream + + +### v2.30.1 + +- support SET NX GET special case + + +### v2.30.0 + +- implement redis 7.0.x (from 6.X). Main changes: + - test against 7.0.7 + - update error messages + - support nx|xx|gt|lt options in [P]EXPIRE[AT] + - update how deleted items are processed in pending queues in streams + + +### v2.23.1 + +- resolve $ to latest ID in XREAD (thanks @josh-hook) +- handle disconnect in blocking functions (thanks @jgirtakovskis) +- fix type conversion bug in redisToLua (thanks Sandy Harvie) +- BRPOP{LPUSH} timeout can be float since 6.0 + + +### v2.23.0 + +- basic INFO support (thanks @kirill-a-belov) +- support COUNT in SSCAN (thanks @Abdi-dd) +- test and support Go 1.19 +- support LPOS (thanks @ianstarz) +- support XPENDING, XGROUP {CREATECONSUMER,DESTROY,DELCONSUMER}, XINFO {CONSUMERS,GROUPS}, XCLAIM (thanks @sandyharvie) + + +### v2.22.0 + +- set miniredis.DumpMaxLineLen to get more Dump() info (thanks @afjoseph) +- fix invalid resposne of COMMAND (thanks @zsh1995) +- fix possibility to generate duplicate IDs in XADD (thanks @readams) +- adds support for XAUTOCLAIM min-idle parameter (thanks @readams) + + +### v2.21.0 + +- support for GETEX (thanks @dntj) +- support for GT and LT in ZADD (thanks @lsgndln) +- support for XAUTOCLAIM (thanks @randall-fulton) + + +### v2.20.0 + +- back to support Go >= 1.14 (thanks @ajatprabha and @marcind) + + +### v2.19.0 + +- support for TYPE in SCAN (thanks @0xDiddi) +- update BITPOS (thanks @dirkm) +- fix a lua redis.call() return value (thanks @mpetronic) +- update ZRANGE (thanks @valdemarpereira) + + +### v2.18.0 + +- support for ZUNION (thanks @propan) +- support for COPY (thanks @matiasinsaurralde and @rockitbaby) +- support for LMOVE (thanks @btwear) + + +### v2.17.0 + +- added miniredis.RunT(t) + + +### v2.16.1 + +- fix ZINTERSTORE with sets (thanks @lingjl2010 and @okhowang) +- fix exclusive ranges in XRANGE (thanks @joseotoro) + + +### v2.16.0 + +- simplify some code (thanks @zonque) +- support for EXAT/PXAT in SET +- support for XTRIM (thanks @joseotoro) +- support for ZRANDMEMBER +- support for redis.log() in lua (thanks @dirkm) + + +### v2.15.2 + +- Fix race condition in blocking code (thanks @zonque and @robx) +- XREAD accepts '$' as ID (thanks @bradengroom) + + +### v2.15.1 + +- EVAL should cache the script (thanks @guoshimin) + + +### v2.15.0 + +- target redis 6.2 and added new args to various commands +- support for all hyperlog commands (thanks @ilbaktin) +- support for GETDEL (thanks @wszaranski) + + +### v2.14.5 + +- added XPENDING +- support for BLOCK option in XREAD and XREADGROUP + + +### v2.14.4 + +- fix BITPOS error (thanks @xiaoyuzdy) +- small fixes for XREAD, XACK, and XDEL. Mostly error cases. +- fix empty EXEC return type (thanks @ashanbrown) +- fix XDEL (thanks @svakili and @yvesf) +- fix FLUSHALL for streams (thanks @svakili) + + +### v2.14.3 + +- fix problem where Lua code didn't set the selected DB +- update to redis 6.0.10 (thanks @lazappa) + + +### v2.14.2 + +- update LUA dependency +- deal with (p)unsubscribe when there are no channels + + +### v2.14.1 + +- mod tidy + + +### v2.14.0 + +- support for HELLO and the RESP3 protocol +- KEEPTTL in SET (thanks @johnpena) + + +### v2.13.3 + +- support Go 1.14 and 1.15 +- update the `Check...()` methods +- support for XREAD (thanks @pieterlexis) + + +### v2.13.2 + +- Use SAN instead of CN in self signed cert for testing (thanks @johejo) +- Travis CI now tests against the most recent two versions of Go (thanks @johejo) +- changed unit and integration tests to compare raw payloads, not parsed payloads +- remove "redigo" dependency + + +### v2.13.1 + +- added HSTRLEN +- minimal support for ACL users in AUTH + + +### v2.13.0 + +- added RunTLS(...) +- added SetError(...) + + +### v2.12.0 + +- redis 6 +- Lua json update (thanks @gsmith85) +- CLUSTER commands (thanks @kratisto) +- fix TOUCH +- fix a shutdown race condition + + +### v2.11.4 + +- ZUNIONSTORE now supports standard set types (thanks @wshirey) + + +### v2.11.3 + +- support for TOUCH (thanks @cleroux) +- support for cluster and stream commands (thanks @kak-tus) + + +### v2.11.2 + +- make sure Lua code is executed concurrently +- add command GEORADIUSBYMEMBER (thanks @kyeett) + + +### v2.11.1 + +- globals protection for Lua code (thanks @vk-outreach) +- HSET update (thanks @carlgreen) +- fix BLPOP block on shutdown (thanks @Asalle) + + +### v2.11.0 + +- added XRANGE/XREVRANGE, XADD, and XLEN (thanks @skateinmars) +- added GEODIST +- improved precision for geohashes, closer to what real redis does +- use 128bit floats internally for INCRBYFLOAT and related (thanks @timnd) + + +### v2.10.1 + +- added m.Server() + + +### v2.10.0 + +- added UNLINK +- fix DEL zero-argument case +- cleanup some direct access commands +- added GEOADD, GEOPOS, GEORADIUS, and GEORADIUS_RO + + +### v2.9.1 + +- fix issue with ZRANGEBYLEX +- fix issue with BRPOPLPUSH and direct access + + +### v2.9.0 + +- proper versioned import of github.com/gomodule/redigo (thanks @yfei1) +- fix messages generated by PSUBSCRIBE +- optional internal seed (thanks @zikaeroh) + + +### v2.8.0 + +Proper `v2` in go.mod. + + +### older + +See https://github.com/alicebob/miniredis/releases for the full changelog diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/LICENSE b/scraper-go/vendor/github.com/alicebob/miniredis/v2/LICENSE new file mode 100644 index 00000000..bb02657c --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Harmen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/Makefile b/scraper-go/vendor/github.com/alicebob/miniredis/v2/Makefile new file mode 100644 index 00000000..2b5ec3ec --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/Makefile @@ -0,0 +1,33 @@ +.PHONY: test +test: ### Run unit tests + go test ./... + +.PHONY: testrace +testrace: ### Run unit tests with race detector + go test -race ./... + +.PHONY: int +int: ### Run integration tests (doesn't download redis server) + ${MAKE} -C integration int + +.PHONY: ci +ci: ### Run full tests suite (including download and compilation of proper redis server) + ${MAKE} test + ${MAKE} -C integration redis_src/redis-server int + ${MAKE} testrace + +.PHONY: clean +clean: ### Clean integration test files and remove compiled redis from integration/redis_src + ${MAKE} -C integration clean + +.PHONY: help +help: +ifeq ($(UNAME), Linux) + @grep -P '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' +else + @# this is not tested, but prepared in advance for you, Mac drivers + @awk -F ':.*###' '$$0 ~ FS {printf "%15s%s\n", $$1 ":", $$2}' \ + $(MAKEFILE_LIST) | grep -v '@awk' | sort +endif + diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/README.md b/scraper-go/vendor/github.com/alicebob/miniredis/v2/README.md new file mode 100644 index 00000000..78e4d324 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/README.md @@ -0,0 +1,344 @@ +# Miniredis + +Pure Go Redis test server, used in Go unittests. + + +## + +Sometimes you want to test code which uses Redis, without making it a full-blown +integration test. +Miniredis implements (parts of) the Redis server, to be used in unittests. It +enables a simple, cheap, in-memory, Redis replacement, with a real TCP interface. Think of it as the Redis version of `net/http/httptest`. + +It saves you from using mock code, and since the redis server lives in the +test process you can query for values directly, without going through the server +stack. + +There are no dependencies on external binaries, so you can easily integrate it in automated build processes. + +Be sure to import v2: +``` +import "github.com/alicebob/miniredis/v2" +``` + +## Commands + +Implemented commands: + + - Connection + - AUTH -- see RequireAuth() + - ECHO + - HELLO -- see RequireUserAuth() + - PING + - SELECT + - SWAPDB + - QUIT + - Key + - COPY + - DEL + - DUMP -- partly, only handles string keys + - EXISTS + - EXPIRE + - EXPIREAT + - EXPIRETIME + - KEYS + - MOVE + - PERSIST + - PEXPIRE + - PEXPIREAT + - PEXPIRETIME + - PTTL + - RANDOMKEY -- see m.Seed(...) + - RENAME + - RENAMENX + - RESTORE -- partly, only handles string keys + - SCAN + - TOUCH + - TTL + - TYPE + - UNLINK + - WAIT -- no-op + - Transactions + - DISCARD + - EXEC + - MULTI + - UNWATCH + - WATCH + - Server + - DBSIZE + - FLUSHALL + - FLUSHDB + - TIME -- returns time.Now() or value set by SetTime() + - COMMAND -- partly + - INFO -- partly, returns only "clients" section with one field "connected_clients" + - String keys + - APPEND + - BITCOUNT + - BITOP + - BITPOS + - DECR + - DECRBY + - DELEX -- partly + - GET + - GETBIT + - GETDEL + - GETEX + - GETRANGE + - GETSET + - INCR + - INCRBY + - INCRBYFLOAT + - MGET + - MSET + - MSETNX + - PSETEX + - SET + - SETBIT + - SETEX + - SETNX + - SETRANGE + - STRLEN + - Hash keys + - HDEL + - HEXISTS + - HGET + - HGETALL + - HINCRBY + - HINCRBYFLOAT + - HKEYS + - HLEN + - HMGET + - HMSET + - HRANDFIELD + - HSET + - HSETNX + - HSTRLEN + - HVALS + - HSCAN + - List keys + - BLPOP + - BRPOP + - BRPOPLPUSH + - LINDEX + - LINSERT + - LLEN + - LPOP + - LPUSH + - LPUSHX + - LRANGE + - LREM + - LSET + - LTRIM + - RPOP + - RPOPLPUSH + - RPUSH + - RPUSHX + - LMOVE + - BLMOVE + - Pub/Sub + - PSUBSCRIBE + - PUBLISH + - PUBSUB + - PUNSUBSCRIBE + - SUBSCRIBE + - UNSUBSCRIBE + - Set keys + - SADD + - SCARD + - SDIFF + - SDIFFSTORE + - SINTER + - SINTERSTORE + - SINTERCARD + - SISMEMBER + - SMEMBERS + - SMISMEMBER + - SMOVE + - SPOP -- see m.Seed(...) + - SRANDMEMBER -- see m.Seed(...) + - SREM + - SSCAN + - SUNION + - SUNIONSTORE + - Sorted Set keys + - ZADD + - ZCARD + - ZCOUNT + - ZINCRBY + - ZINTER + - ZINTERSTORE + - ZLEXCOUNT + - ZPOPMIN + - ZPOPMAX + - ZRANDMEMBER + - ZRANGE + - ZRANGEBYLEX + - ZRANGEBYSCORE + - ZRANK + - ZREM + - ZREMRANGEBYLEX + - ZREMRANGEBYRANK + - ZREMRANGEBYSCORE + - ZREVRANGE + - ZREVRANGEBYLEX + - ZREVRANGEBYSCORE + - ZREVRANK + - ZSCORE + - ZUNION + - ZUNIONSTORE + - ZSCAN + - Stream keys + - XACK + - XADD + - XAUTOCLAIM + - XCLAIM + - XDEL + - XGROUP CREATE + - XGROUP CREATECONSUMER + - XGROUP DESTROY + - XGROUP DELCONSUMER + - XINFO STREAM -- partly + - XINFO GROUPS + - XINFO CONSUMERS -- partly + - XLEN + - XRANGE + - XREAD + - XREADGROUP + - XREVRANGE + - XPENDING + - XTRIM + - Scripting + - EVAL + - EVALSHA + - SCRIPT LOAD + - SCRIPT EXISTS + - SCRIPT FLUSH + - GEO + - GEOADD + - GEODIST + - ~~GEOHASH~~ + - GEOPOS + - GEORADIUS + - GEORADIUS_RO + - GEORADIUSBYMEMBER + - GEORADIUSBYMEMBER_RO + - Cluster + - CLUSTER SLOTS + - CLUSTER KEYSLOT + - CLUSTER NODES + - CLUSTER SHARDS + - HyperLogLog + - PFADD + - PFCOUNT + - PFMERGE + + +## TTLs, key expiration, and time + +Since miniredis is intended to be used in unittests TTLs don't decrease +automatically. You can use `TTL()` to get the TTL (as a time.Duration) of a +key. It will return 0 when no TTL is set. + +`m.FastForward(d)` can be used to decrement all TTLs. All TTLs which become <= +0 will be removed. + +EXPIREAT and PEXPIREAT values will be +converted to a duration. For that you can either set m.SetTime(t) to use that +time as the base for the (P)EXPIREAT conversion, or don't call SetTime(), in +which case time.Now() will be used. + +SetTime() also sets the value returned by TIME, which defaults to time.Now(). +It is not updated by FastForward, only by SetTime. + +## Randomness and Seed() + +Miniredis will use `math/rand`'s global RNG for randomness unless a seed is +provided by calling `m.Seed(...)`. If a seed is provided, then miniredis will +use its own RNG based on that seed. + +Commands which use randomness are: RANDOMKEY, SPOP, and SRANDMEMBER. + +## Example + +``` Go + +import ( + ... + "github.com/alicebob/miniredis/v2" + ... +) + +func TestSomething(t *testing.T) { + s := miniredis.RunT(t) + + // Optionally set some keys your code expects: + s.Set("foo", "bar") + s.HSet("some", "other", "key") + + // Run your code and see if it behaves. + // An example using the redigo library from "github.com/gomodule/redigo/redis": + c, err := redis.Dial("tcp", s.Addr()) + _, err = c.Do("SET", "foo", "bar") + + // Optionally check values in redis... + if got, err := s.Get("foo"); err != nil || got != "bar" { + t.Error("'foo' has the wrong value") + } + // ... or use a helper for that: + s.CheckGet(t, "foo", "bar") + + // TTL and expiration: + s.Set("foo", "bar") + s.SetTTL("foo", 10*time.Second) + s.FastForward(11 * time.Second) + if s.Exists("foo") { + t.Fatal("'foo' should not have existed anymore") + } +} +``` + +## Not supported + +Commands which will probably not be implemented: + + - CLUSTER (all) + - ~~CLUSTER *~~ + - ~~READONLY~~ + - ~~READWRITE~~ + - Key + - ~~MIGRATE~~ + - ~~OBJECT~~ + - Scripting + - ~~FCALL / FCALL_RO *~~ + - ~~FUNCTION *~~ + - ~~SCRIPT DEBUG~~ + - ~~SCRIPT KILL~~ + - Server + - ~~BGSAVE~~ + - ~~BGWRITEAOF~~ + - ~~CLIENT *~~ + - ~~CONFIG *~~ + - ~~DEBUG *~~ + - ~~LASTSAVE~~ + - ~~MONITOR~~ + - ~~ROLE~~ + - ~~SAVE~~ + - ~~SHUTDOWN~~ + - ~~SLAVEOF~~ + - ~~SLOWLOG~~ + - ~~SYNC~~ + + +## &c. + +Integration tests are run against Redis 8.4.0. The [./integration](./integration/) subdir +compares miniredis against a real redis instance. + +The Redis 6 RESP3 protocol is supported. If there are problems, please open +an issue. + +If you want to test Redis Sentinel have a look at [minisentinel](https://github.com/Bose/minisentinel). + +A changelog is kept at [CHANGELOG.md](https://github.com/alicebob/miniredis/blob/master/CHANGELOG.md). + +[![Go Reference](https://pkg.go.dev/badge/github.com/alicebob/miniredis/v2.svg)](https://pkg.go.dev/github.com/alicebob/miniredis/v2) diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/check.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/check.go new file mode 100644 index 00000000..acd0d553 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/check.go @@ -0,0 +1,63 @@ +package miniredis + +import ( + "reflect" + "sort" +) + +// T is implemented by Testing.T +type T interface { + Helper() + Errorf(string, ...interface{}) +} + +// CheckGet does not call Errorf() iff there is a string key with the +// expected value. Normal use case is `m.CheckGet(t, "username", "theking")`. +func (m *Miniredis) CheckGet(t T, key, expected string) { + t.Helper() + + found, err := m.Get(key) + if err != nil { + t.Errorf("GET error, key %#v: %v", key, err) + return + } + if found != expected { + t.Errorf("GET error, key %#v: Expected %#v, got %#v", key, expected, found) + return + } +} + +// CheckList does not call Errorf() iff there is a list key with the +// expected values. +// Normal use case is `m.CheckGet(t, "favorite_colors", "red", "green", "infrared")`. +func (m *Miniredis) CheckList(t T, key string, expected ...string) { + t.Helper() + + found, err := m.List(key) + if err != nil { + t.Errorf("List error, key %#v: %v", key, err) + return + } + if !reflect.DeepEqual(expected, found) { + t.Errorf("List error, key %#v: Expected %#v, got %#v", key, expected, found) + return + } +} + +// CheckSet does not call Errorf() iff there is a set key with the +// expected values. +// Normal use case is `m.CheckSet(t, "visited", "Rome", "Stockholm", "Dublin")`. +func (m *Miniredis) CheckSet(t T, key string, expected ...string) { + t.Helper() + + found, err := m.Members(key) + if err != nil { + t.Errorf("Set error, key %#v: %v", key, err) + return + } + sort.Strings(expected) + if !reflect.DeepEqual(expected, found) { + t.Errorf("Set error, key %#v: Expected %#v, got %#v", key, expected, found) + return + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_client.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_client.go new file mode 100644 index 00000000..ca9fcd9a --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_client.go @@ -0,0 +1,68 @@ +package miniredis + +import ( + "fmt" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsClient handles client operations. +func commandsClient(m *Miniredis) { + m.srv.Register("CLIENT", m.cmdClient) +} + +// CLIENT +func (m *Miniredis) cmdClient(c *server.Peer, cmd string, args []string) { + if len(args) == 0 { + setDirty(c) + c.WriteError("ERR wrong number of arguments for 'client' command") + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + switch cmd := strings.ToUpper(args[0]); cmd { + case "SETNAME": + m.cmdClientSetName(c, args[1:]) + case "GETNAME": + m.cmdClientGetName(c, args[1:]) + default: + setDirty(c) + c.WriteError(fmt.Sprintf("ERR unknown subcommand '%s'. Try CLIENT HELP.", cmd)) + } + }) +} + +// CLIENT SETNAME +func (m *Miniredis) cmdClientSetName(c *server.Peer, args []string) { + if len(args) != 1 { + setDirty(c) + c.WriteError("ERR wrong number of arguments for 'client setname' command") + return + } + + name := args[0] + if strings.ContainsAny(name, " \n") { + setDirty(c) + c.WriteError("ERR Client names cannot contain spaces, newlines or special characters.") + return + + } + c.ClientName = name + c.WriteOK() +} + +// CLIENT GETNAME +func (m *Miniredis) cmdClientGetName(c *server.Peer, args []string) { + if len(args) > 0 { + setDirty(c) + c.WriteError("ERR wrong number of arguments for 'client getname' command") + return + } + + if c.ClientName == "" { + c.WriteNull() + } else { + c.WriteBulk(c.ClientName) + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_cluster.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_cluster.go new file mode 100644 index 00000000..4f7c77f4 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_cluster.go @@ -0,0 +1,122 @@ +// Commands from https://redis.io/commands#cluster + +package miniredis + +import ( + "fmt" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsCluster handles some cluster operations. +func commandsCluster(m *Miniredis) { + m.srv.Register("CLUSTER", m.cmdCluster) +} + +func (m *Miniredis) cmdCluster(c *server.Peer, cmd string, args []string) { + if !m.handleAuth(c) { + return + } + + if len(args) < 1 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + switch strings.ToUpper(args[0]) { + case "SLOTS": + m.cmdClusterSlots(c, cmd, args) + case "KEYSLOT": + m.cmdClusterKeySlot(c, cmd, args) + case "NODES": + m.cmdClusterNodes(c, cmd, args) + case "SHARDS": + m.cmdClusterShards(c, cmd, args) + default: + setDirty(c) + c.WriteError(fmt.Sprintf("ERR 'CLUSTER %s' not supported", strings.Join(args, " "))) + return + } +} + +// CLUSTER SLOTS +func (m *Miniredis) cmdClusterSlots(c *server.Peer, cmd string, args []string) { + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + c.WriteLen(1) + c.WriteLen(3) + c.WriteInt(0) + c.WriteInt(16383) + c.WriteLen(3) + c.WriteBulk(m.srv.Addr().IP.String()) + c.WriteInt(m.srv.Addr().Port) + c.WriteBulk("09dbe9720cda62f7865eabc5fd8857c5d2678366") + }) +} + +// CLUSTER KEYSLOT +func (m *Miniredis) cmdClusterKeySlot(c *server.Peer, cmd string, args []string) { + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + c.WriteInt(163) + }) +} + +// CLUSTER NODES +func (m *Miniredis) cmdClusterNodes(c *server.Peer, cmd string, args []string) { + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + // do not try to use m.Addr() here, as m is blocked by this tx. + addr := m.srv.Addr() + port := m.srv.Addr().Port + c.WriteBulk(fmt.Sprintf("e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca %s@%d myself,master - 0 0 1 connected 0-16383", addr, port)) + }) +} + +// CLUSTER SHARDS +func (m *Miniredis) cmdClusterShards(c *server.Peer, cmd string, args []string) { + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + addr := m.srv.Addr() + host := addr.IP.String() + port := addr.Port + + // Array of shards (we return 1 shard) + c.WriteLen(1) + + // Shard is a map with 2 keys: "slots" and "nodes" + c.WriteMapLen(2) + + // "slots": flat list of start/end pairs (inclusive ranges) + c.WriteBulk("slots") + c.WriteLen(2) + c.WriteInt(0) + c.WriteInt(16383) + + // "nodes": array of node maps + c.WriteBulk("nodes") + c.WriteLen(1) + + // Node map. + // (id, endpoint, ip, port, role, replication-offset, health) + c.WriteMapLen(6) + + c.WriteBulk("id") + c.WriteBulk("13f84e686106847b76671957dd348fde540a77bb") + + //c.WriteBulk("endpoint") + //c.WriteBulk(host) // or host:port if your client expects that + + c.WriteBulk("ip") + c.WriteBulk(host) + + c.WriteBulk("port") + c.WriteInt(port) + + c.WriteBulk("role") + c.WriteBulk("master") + + c.WriteBulk("replication-offset") + c.WriteInt(0) + + c.WriteBulk("health") + c.WriteBulk("online") + }) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_command.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_command.go new file mode 100644 index 00000000..8f73b2ba --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_command.go @@ -0,0 +1,14 @@ +// Command 'COMMAND' from https://redis.io/commands#server + +package miniredis + +import "github.com/alicebob/miniredis/v2/server" + +func (m *Miniredis) cmdCommand(c *server.Peer, cmd string, args []string) { + // Got from redis 5.0.7 with + // echo 'COMMAND' | nc redis_addr redis_port + + res := "*200\r\n*6\r\n$12\r\nhincrbyfloat\r\n:4\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$10\r\nxreadgroup\r\n:-7\r\n*3\r\n+write\r\n+noscript\r\n+movablekeys\r\n:1\r\n:1\r\n:1\r\n*6\r\n$10\r\nsdiffstore\r\n:-3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$8\r\nlastsave\r\n:1\r\n*2\r\n+random\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nsetnx\r\n:3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$8\r\nbzpopmax\r\n:-3\r\n*3\r\n+write\r\n+noscript\r\n+fast\r\n:1\r\n:-2\r\n:1\r\n*6\r\n$12\r\npunsubscribe\r\n:-1\r\n*4\r\n+pubsub\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nxack\r\n:-4\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$10\r\npfselftest\r\n:1\r\n*1\r\n+admin\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nsubstr\r\n:4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$8\r\nsmembers\r\n:2\r\n*2\r\n+readonly\r\n+sort_for_script\r\n:1\r\n:1\r\n:1\r\n*6\r\n$11\r\nunsubscribe\r\n:-1\r\n*4\r\n+pubsub\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$11\r\nzinterstore\r\n:-4\r\n*3\r\n+write\r\n+denyoom\r\n+movablekeys\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nstrlen\r\n:2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\npfmerge\r\n:-2\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$9\r\nrandomkey\r\n:1\r\n*2\r\n+readonly\r\n+random\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nlolwut\r\n:-1\r\n*1\r\n+readonly\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nrpop\r\n:2\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nhkeys\r\n:2\r\n*2\r\n+readonly\r\n+sort_for_script\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nclient\r\n:-2\r\n*2\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nmodule\r\n:-2\r\n*2\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\nslowlog\r\n:-2\r\n*2\r\n+admin\r\n+random\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\ngeohash\r\n:-2\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nlrange\r\n:4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nping\r\n:-1\r\n*2\r\n+stale\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$8\r\nbitcount\r\n:-2\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\npubsub\r\n:-2\r\n*4\r\n+pubsub\r\n+random\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nrole\r\n:1\r\n*3\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nhget\r\n:3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nobject\r\n:-2\r\n*2\r\n+readonly\r\n+random\r\n:2\r\n:2\r\n:1\r\n*6\r\n$9\r\nzrevrange\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nhincrby\r\n:4\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$9\r\nzlexcount\r\n:4\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nscard\r\n:2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nappend\r\n:3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nhstrlen\r\n:3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nconfig\r\n:-2\r\n*4\r\n+admin\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nhset\r\n:-4\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$16\r\nzrevrangebyscore\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nincr\r\n:2\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nsetbit\r\n:4\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$9\r\nrpoplpush\r\n:3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:2\r\n:1\r\n*6\r\n$6\r\nxclaim\r\n:-6\r\n*3\r\n+write\r\n+random\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$11\r\nsinterstore\r\n:-3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$7\r\npublish\r\n:3\r\n*4\r\n+pubsub\r\n+loading\r\n+stale\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nhscan\r\n:-3\r\n*2\r\n+readonly\r\n+random\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nmulti\r\n:1\r\n*2\r\n+noscript\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$3\r\nset\r\n:-3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nlpushx\r\n:-3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$16\r\nzremrangebyscore\r\n:4\r\n*1\r\n+write\r\n:1\r\n:1\r\n:1\r\n*6\r\n$9\r\npexpireat\r\n:3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nhdel\r\n:-3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$12\r\nbgrewriteaof\r\n:1\r\n*2\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\nmigrate\r\n:-6\r\n*3\r\n+write\r\n+random\r\n+movablekeys\r\n:0\r\n:0\r\n:0\r\n*6\r\n$9\r\nreplicaof\r\n:3\r\n*3\r\n+admin\r\n+noscript\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\ntouch\r\n:-2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nxsetid\r\n:3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nbitop\r\n:-4\r\n*2\r\n+write\r\n+denyoom\r\n:2\r\n:-1\r\n:1\r\n*6\r\n$6\r\nswapdb\r\n:3\r\n*2\r\n+write\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nsdiff\r\n:-2\r\n*2\r\n+readonly\r\n+sort_for_script\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$6\r\nlindex\r\n:3\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nwait\r\n:3\r\n*1\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nlrem\r\n:4\r\n*1\r\n+write\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nhsetnx\r\n:4\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$8\r\ngetrange\r\n:4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nhlen\r\n:2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\npost\r\n:-1\r\n*2\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$9\r\nsismember\r\n:3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nunwatch\r\n:1\r\n*2\r\n+noscript\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nlpush\r\n:-3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nscan\r\n:-2\r\n*2\r\n+readonly\r\n+random\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nsmove\r\n:4\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:2\r\n:1\r\n*6\r\n$7\r\ncluster\r\n:-2\r\n*1\r\n+admin\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nbgsave\r\n:-1\r\n*2\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\ndump\r\n:2\r\n*2\r\n+readonly\r\n+random\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nlatency\r\n:-2\r\n*4\r\n+admin\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$8\r\nbzpopmin\r\n:-3\r\n*3\r\n+write\r\n+noscript\r\n+fast\r\n:1\r\n:-2\r\n:1\r\n*6\r\n$6\r\ngetbit\r\n:3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nhgetall\r\n:2\r\n*2\r\n+readonly\r\n+random\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nrename\r\n:3\r\n*1\r\n+write\r\n:1\r\n:2\r\n:1\r\n*6\r\n$9\r\nsubscribe\r\n:-2\r\n*4\r\n+pubsub\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nxdel\r\n:-3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$15\r\nzremrangebyrank\r\n:4\r\n*1\r\n+write\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\ntype\r\n:2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nscript\r\n:-2\r\n*1\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nhmset\r\n:-4\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nsunion\r\n:-2\r\n*2\r\n+readonly\r\n+sort_for_script\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$4\r\nmget\r\n:-2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$10\r\nbrpoplpush\r\n:4\r\n*3\r\n+write\r\n+denyoom\r\n+noscript\r\n:1\r\n:2\r\n:1\r\n*6\r\n$6\r\ngeoadd\r\n:-5\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\ndecrby\r\n:3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\necho\r\n:2\r\n*1\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\ndbsize\r\n:1\r\n*2\r\n+readonly\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nzcard\r\n:2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nselect\r\n:2\r\n*2\r\n+loading\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nsadd\r\n:-3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nhost:\r\n:-1\r\n*2\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nsscan\r\n:-3\r\n*2\r\n+readonly\r\n+random\r\n:1\r\n:1\r\n:1\r\n*6\r\n$12\r\ngeoradius_ro\r\n:-6\r\n*2\r\n+readonly\r\n+movablekeys\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nmonitor\r\n:1\r\n*2\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$14\r\nzremrangebylex\r\n:4\r\n*1\r\n+write\r\n:1\r\n:1\r\n:1\r\n*6\r\n$11\r\nsunionstore\r\n:-3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$5\r\nzscan\r\n:-3\r\n*2\r\n+readonly\r\n+random\r\n:1\r\n:1\r\n:1\r\n*6\r\n$9\r\nreadwrite\r\n:1\r\n*1\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nxgroup\r\n:-2\r\n*2\r\n+write\r\n+denyoom\r\n:2\r\n:2\r\n:1\r\n*6\r\n$5\r\nsetex\r\n:4\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nsave\r\n:1\r\n*2\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nhvals\r\n:2\r\n*2\r\n+readonly\r\n+sort_for_script\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nwatch\r\n:-2\r\n*2\r\n+noscript\r\n+fast\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$7\r\nhexists\r\n:3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\ninfo\r\n:-1\r\n*3\r\n+random\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\npsync\r\n:3\r\n*3\r\n+readonly\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$11\r\nzrangebylex\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nzadd\r\n:-4\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nxlen\r\n:2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nauth\r\n:2\r\n*4\r\n+noscript\r\n+loading\r\n+stale\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nsrem\r\n:-3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$9\r\ngeoradius\r\n:-6\r\n*2\r\n+write\r\n+movablekeys\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nexec\r\n:1\r\n*2\r\n+noscript\r\n+skip_monitor\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\npfcount\r\n:-2\r\n*1\r\n+readonly\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$7\r\nzpopmin\r\n:-2\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nmove\r\n:3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nxtrim\r\n:-2\r\n*3\r\n+write\r\n+random\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nasking\r\n:1\r\n*1\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\npttl\r\n:2\r\n*3\r\n+readonly\r\n+random\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$11\r\nsrandmember\r\n:-2\r\n*2\r\n+readonly\r\n+random\r\n:1\r\n:1\r\n:1\r\n*6\r\n$8\r\nflushall\r\n:-1\r\n*1\r\n+write\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nsort\r\n:-2\r\n*3\r\n+write\r\n+denyoom\r\n+movablekeys\r\n:1\r\n:1\r\n:1\r\n*6\r\n$3\r\ndel\r\n:-2\r\n*1\r\n+write\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$14\r\nrestore-asking\r\n:-4\r\n*3\r\n+write\r\n+denyoom\r\n+asking\r\n:1\r\n:1\r\n:1\r\n*6\r\n$10\r\npsubscribe\r\n:-2\r\n*4\r\n+pubsub\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\ndecr\r\n:2\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nincrby\r\n:3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$14\r\nzrevrangebylex\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$8\r\nbitfield\r\n:-2\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nexists\r\n:-2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$8\r\nreplconf\r\n:-1\r\n*4\r\n+admin\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\nzincrby\r\n:4\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nblpop\r\n:-3\r\n*2\r\n+write\r\n+noscript\r\n:1\r\n:-2\r\n:1\r\n*6\r\n$4\r\nlpop\r\n:2\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$3\r\nttl\r\n:2\r\n*3\r\n+readonly\r\n+random\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nxread\r\n:-4\r\n*3\r\n+readonly\r\n+noscript\r\n+movablekeys\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nrpush\r\n:-3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$8\r\nzrevrank\r\n:3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$11\r\nincrbyfloat\r\n:3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nbrpop\r\n:-3\r\n*2\r\n+write\r\n+noscript\r\n:1\r\n:-2\r\n:1\r\n*6\r\n$4\r\nxadd\r\n:-5\r\n*4\r\n+write\r\n+denyoom\r\n+random\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$8\r\nsetrange\r\n:4\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$17\r\ngeoradiusbymember\r\n:-5\r\n*2\r\n+write\r\n+movablekeys\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nunlink\r\n:-2\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$8\r\nexpireat\r\n:3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\ndebug\r\n:-2\r\n*2\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$20\r\ngeoradiusbymember_ro\r\n:-5\r\n*2\r\n+readonly\r\n+movablekeys\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nlset\r\n:4\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nzscore\r\n:3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nllen\r\n:2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\ntime\r\n:1\r\n*2\r\n+random\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$8\r\nshutdown\r\n:-1\r\n*4\r\n+admin\r\n+noscript\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\nevalsha\r\n:-3\r\n*2\r\n+noscript\r\n+movablekeys\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nzcount\r\n:4\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nmemory\r\n:-2\r\n*2\r\n+readonly\r\n+random\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nxinfo\r\n:-2\r\n*2\r\n+readonly\r\n+random\r\n:2\r\n:2\r\n:1\r\n*6\r\n$8\r\nxpending\r\n:-3\r\n*2\r\n+readonly\r\n+random\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\neval\r\n:-3\r\n*2\r\n+noscript\r\n+movablekeys\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nxrange\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nrestore\r\n:-4\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nzpopmax\r\n:-2\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nmset\r\n:-3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:-1\r\n:2\r\n*6\r\n$4\r\nspop\r\n:-2\r\n*3\r\n+write\r\n+random\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nltrim\r\n:4\r\n*1\r\n+write\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\nzrank\r\n:3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$9\r\nxrevrange\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$3\r\nget\r\n:2\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nflushdb\r\n:-1\r\n*1\r\n+write\r\n:0\r\n:0\r\n:0\r\n*6\r\n$5\r\nhmget\r\n:-3\r\n*2\r\n+readonly\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nmsetnx\r\n:-3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:-1\r\n:2\r\n*6\r\n$7\r\npersist\r\n:2\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$11\r\nzunionstore\r\n:-4\r\n*3\r\n+write\r\n+denyoom\r\n+movablekeys\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\ncommand\r\n:0\r\n*3\r\n+random\r\n+loading\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$8\r\nrenamenx\r\n:3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:2\r\n:1\r\n*6\r\n$6\r\nzrange\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\npexpire\r\n:3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nkeys\r\n:2\r\n*2\r\n+readonly\r\n+sort_for_script\r\n:0\r\n:0\r\n:0\r\n*6\r\n$4\r\nzrem\r\n:-3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$5\r\npfadd\r\n:-2\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\npsetex\r\n:4\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$13\r\nzrangebyscore\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$4\r\nsync\r\n:1\r\n*3\r\n+readonly\r\n+admin\r\n+noscript\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\npfdebug\r\n:-3\r\n*1\r\n+write\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\ndiscard\r\n:1\r\n*2\r\n+noscript\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$8\r\nreadonly\r\n:1\r\n*1\r\n+fast\r\n:0\r\n:0\r\n:0\r\n*6\r\n$7\r\ngeodist\r\n:-4\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\ngeopos\r\n:-2\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nbitpos\r\n:-3\r\n*1\r\n+readonly\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nsinter\r\n:-2\r\n*2\r\n+readonly\r\n+sort_for_script\r\n:1\r\n:-1\r\n:1\r\n*6\r\n$6\r\ngetset\r\n:3\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nslaveof\r\n:3\r\n*3\r\n+admin\r\n+noscript\r\n+stale\r\n:0\r\n:0\r\n:0\r\n*6\r\n$6\r\nrpushx\r\n:-3\r\n*3\r\n+write\r\n+denyoom\r\n+fast\r\n:1\r\n:1\r\n:1\r\n*6\r\n$7\r\nlinsert\r\n:5\r\n*2\r\n+write\r\n+denyoom\r\n:1\r\n:1\r\n:1\r\n*6\r\n$6\r\nexpire\r\n:3\r\n*2\r\n+write\r\n+fast\r\n:1\r\n:1\r\n:1\r\n" + + c.WriteRaw(res) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_connection.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_connection.go new file mode 100644 index 00000000..b4ec55d7 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_connection.go @@ -0,0 +1,281 @@ +// Commands from https://redis.io/commands#connection + +package miniredis + +import ( + "fmt" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +func commandsConnection(m *Miniredis) { + m.srv.Register("AUTH", m.cmdAuth) + m.srv.Register("ECHO", m.cmdEcho) + m.srv.Register("HELLO", m.cmdHello) + m.srv.Register("PING", m.cmdPing) + m.srv.Register("QUIT", m.cmdQuit) + m.srv.Register("SELECT", m.cmdSelect) + m.srv.Register("SWAPDB", m.cmdSwapdb) +} + +// PING +func (m *Miniredis) cmdPing(c *server.Peer, cmd string, args []string) { + if !m.handleAuth(c) { + return + } + + if len(args) > 1 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + + payload := "" + if len(args) > 0 { + payload = args[0] + } + + // PING is allowed in subscribed state + if sub := getCtx(c).subscriber; sub != nil { + c.Block(func(c *server.Writer) { + c.WriteLen(2) + c.WriteBulk("pong") + c.WriteBulk(payload) + }) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + if payload == "" { + c.WriteInline("PONG") + return + } + c.WriteBulk(payload) + }) +} + +// AUTH +func (m *Miniredis) cmdAuth(c *server.Peer, cmd string, args []string) { + if len(args) < 1 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + + if len(args) > 2 { + c.WriteError(msgSyntaxError) + return + } + if m.checkPubsub(c, cmd) { + return + } + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + + var opts = struct { + username string + password string + }{ + username: "default", + password: args[0], + } + if len(args) == 2 { + opts.username, opts.password = args[0], args[1] + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + if len(m.passwords) == 0 && opts.username == "default" { + c.WriteError("ERR AUTH called without any password configured for the default user. Are you sure your configuration is correct?") + return + } + setPW, ok := m.passwords[opts.username] + if !ok { + c.WriteError("WRONGPASS invalid username-password pair") + return + } + if setPW != opts.password { + c.WriteError("WRONGPASS invalid username-password pair") + return + } + + ctx.authenticated = true + c.WriteOK() + }) +} + +// HELLO +func (m *Miniredis) cmdHello(c *server.Peer, cmd string, args []string) { + if len(args) < 1 { + c.WriteError(errWrongNumber(cmd)) + return + } + + var opts struct { + version int + username string + password string + } + + if ok := optIntErr(c, args[0], &opts.version, "ERR Protocol version is not an integer or out of range"); !ok { + return + } + args = args[1:] + + switch opts.version { + case 2, 3: + default: + c.WriteError("NOPROTO unsupported protocol version") + return + } + + var checkAuth bool + for len(args) > 0 { + switch strings.ToUpper(args[0]) { + case "AUTH": + if len(args) < 3 { + c.WriteError(fmt.Sprintf("ERR Syntax error in HELLO option '%s'", args[0])) + return + } + opts.username, opts.password, args = args[1], args[2], args[3:] + checkAuth = true + case "SETNAME": + if len(args) < 2 { + c.WriteError(fmt.Sprintf("ERR Syntax error in HELLO option '%s'", args[0])) + return + } + _, args = args[1], args[2:] + default: + c.WriteError(fmt.Sprintf("ERR Syntax error in HELLO option '%s'", args[0])) + return + } + } + + if len(m.passwords) == 0 && opts.username == "default" { + // redis ignores legacy "AUTH" if it's not enabled. + checkAuth = false + } + if checkAuth { + setPW, ok := m.passwords[opts.username] + if !ok { + c.WriteError("WRONGPASS invalid username-password pair") + return + } + if setPW != opts.password { + c.WriteError("WRONGPASS invalid username-password pair") + return + } + getCtx(c).authenticated = true + } + + c.Resp3 = opts.version == 3 + + c.WriteMapLen(7) + c.WriteBulk("server") + c.WriteBulk("miniredis") + c.WriteBulk("version") + c.WriteBulk("8.4.0") + c.WriteBulk("proto") + c.WriteInt(opts.version) + c.WriteBulk("id") + c.WriteInt(42) + c.WriteBulk("mode") + c.WriteBulk("standalone") + c.WriteBulk("role") + c.WriteBulk("master") + c.WriteBulk("modules") // "modules": [ + c.WriteLen(1) // we have 1: "vectorset" + c.WriteMapLen(4) // { + c.WriteBulk("name") // + c.WriteBulk("vectorset") // + c.WriteBulk("ver") // + c.WriteInt(1) // + c.WriteBulk("path") // + c.WriteBulk("") // + c.WriteBulk("args") // + c.WriteLen(0) // ]} end modules +} + +// ECHO +func (m *Miniredis) cmdEcho(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + msg := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + c.WriteBulk(msg) + }) +} + +// SELECT +func (m *Miniredis) cmdSelect(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + var opts struct { + id int + } + if ok := optInt(c, args[0], &opts.id); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + if opts.id < 0 { + c.WriteError(msgDBIndexOutOfRange) + setDirty(c) + return + } + + ctx.selectedDB = opts.id + c.WriteOK() + }) +} + +// SWAPDB +func (m *Miniredis) cmdSwapdb(c *server.Peer, cmd string, args []string) { + if len(args) != 2 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + if !m.handleAuth(c) { + return + } + + var opts struct { + id1 int + id2 int + } + + if ok := optIntErr(c, args[0], &opts.id1, "ERR invalid first DB index"); !ok { + return + } + if ok := optIntErr(c, args[1], &opts.id2, "ERR invalid second DB index"); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + if opts.id1 < 0 || opts.id2 < 0 { + c.WriteError(msgDBIndexOutOfRange) + setDirty(c) + return + } + + m.swapDB(opts.id1, opts.id2) + + c.WriteOK() + }) +} + +// QUIT +func (m *Miniredis) cmdQuit(c *server.Peer, cmd string, args []string) { + // QUIT isn't transactionfied and accepts any arguments. + c.WriteOK() + c.Close() +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_generic.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_generic.go new file mode 100644 index 00000000..08c47a28 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_generic.go @@ -0,0 +1,789 @@ +// Commands from https://redis.io/commands#generic + +package miniredis + +import ( + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/alicebob/miniredis/v2/server" +) + +const ( + // expiretimeReplyNoExpiration is return value for EXPIRETIME and PEXPIRETIME if the key exists but has no associated expiration time + expiretimeReplyNoExpiration = -1 + // expiretimeReplyMissingKey is return value for EXPIRETIME and PEXPIRETIME if the key does not exist + expiretimeReplyMissingKey = -2 +) + +func inSeconds(t time.Time) int { + return int(t.Unix()) +} + +func inMilliSeconds(t time.Time) int { + return int(t.UnixMilli()) +} + +// commandsGeneric handles EXPIRE, TTL, PERSIST, &c. +func commandsGeneric(m *Miniredis) { + m.srv.Register("COPY", m.cmdCopy) + m.srv.Register("DEL", m.cmdDel) + m.srv.Register("DUMP", m.cmdDump, server.ReadOnlyOption()) + m.srv.Register("EXISTS", m.cmdExists, server.ReadOnlyOption()) + m.srv.Register("EXPIRE", makeCmdExpire(m, false, time.Second)) + m.srv.Register("EXPIREAT", makeCmdExpire(m, true, time.Second)) + m.srv.Register("EXPIRETIME", m.makeCmdExpireTime(inSeconds), server.ReadOnlyOption()) + m.srv.Register("PEXPIRETIME", m.makeCmdExpireTime(inMilliSeconds), server.ReadOnlyOption()) + m.srv.Register("KEYS", m.cmdKeys, server.ReadOnlyOption()) + // MIGRATE + m.srv.Register("MOVE", m.cmdMove) + // OBJECT + m.srv.Register("PERSIST", m.cmdPersist) + m.srv.Register("PEXPIRE", makeCmdExpire(m, false, time.Millisecond)) + m.srv.Register("PEXPIREAT", makeCmdExpire(m, true, time.Millisecond)) + m.srv.Register("PTTL", m.cmdPTTL, server.ReadOnlyOption()) + m.srv.Register("RANDOMKEY", m.cmdRandomkey, server.ReadOnlyOption()) + m.srv.Register("RENAME", m.cmdRename) + m.srv.Register("RENAMENX", m.cmdRenamenx) + m.srv.Register("RESTORE", m.cmdRestore) + m.srv.Register("TOUCH", m.cmdTouch, server.ReadOnlyOption()) + m.srv.Register("TTL", m.cmdTTL, server.ReadOnlyOption()) + m.srv.Register("TYPE", m.cmdType, server.ReadOnlyOption()) + m.srv.Register("SCAN", m.cmdScan, server.ReadOnlyOption()) + // SORT + m.srv.Register("UNLINK", m.cmdDel) + m.srv.Register("WAIT", m.cmdWait) +} + +type expireOpts struct { + key string + value int + nx bool + xx bool + gt bool + lt bool +} + +func expireParse(cmd string, args []string) (*expireOpts, error) { + var opts expireOpts + + opts.key = args[0] + if err := optIntSimple(args[1], &opts.value); err != nil { + return nil, err + } + args = args[2:] + for len(args) > 0 { + switch strings.ToLower(args[0]) { + case "nx": + opts.nx = true + case "xx": + opts.xx = true + case "gt": + opts.gt = true + case "lt": + opts.lt = true + default: + return nil, fmt.Errorf("ERR Unsupported option %s", args[0]) + } + args = args[1:] + } + if opts.gt && opts.lt { + return nil, errors.New("ERR GT and LT options at the same time are not compatible") + } + if opts.nx && (opts.xx || opts.gt || opts.lt) { + return nil, errors.New("ERR NX and XX, GT or LT options at the same time are not compatible") + } + return &opts, nil +} + +// generic expire command for EXPIRE, PEXPIRE, EXPIREAT, PEXPIREAT +// d is the time unit. If unix is set it'll be seen as a unixtimestamp and +// converted to a duration. +func makeCmdExpire(m *Miniredis, unix bool, d time.Duration) func(*server.Peer, string, []string) { + return func(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + opts, err := expireParse(cmd, args) + if err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + // Key must be present. + if _, ok := db.keys[opts.key]; !ok { + c.WriteInt(0) + return + } + + oldTTL, ok := db.ttl[opts.key] + + var newTTL time.Duration + if unix { + newTTL = m.at(opts.value, d) + } else { + newTTL = time.Duration(opts.value) * d + } + + // > NX -- Set expiry only when the key has no expiry + if opts.nx && ok { + c.WriteInt(0) + return + } + // > XX -- Set expiry only when the key has an existing expiry + if opts.xx && !ok { + c.WriteInt(0) + return + } + // > GT -- Set expiry only when the new expiry is greater than current one + // (no exp == infinity) + if opts.gt && (!ok || newTTL <= oldTTL) { + c.WriteInt(0) + return + } + // > LT -- Set expiry only when the new expiry is less than current one + if opts.lt && ok && newTTL > oldTTL { + c.WriteInt(0) + return + } + db.ttl[opts.key] = newTTL + db.incr(opts.key) + db.checkTTL(opts.key) + c.WriteInt(1) + }) + } +} + +// makeCmdExpireTime creates server command function that returns the absolute Unix timestamp (since January 1, 1970) +// at which the given key will expire, in unit selected by time result strategy (e.g. seconds, milliseconds). +// For more information see redis documentation for [expiretime] and [pexpiretime]. +// +// [expiretime]: https://redis.io/commands/expiretime/ +// [pexpiretime]: https://redis.io/commands/pexpiretime/ +func (m *Miniredis) makeCmdExpireTime(timeResultStrategy func(time.Time) int) server.Cmd { + return func(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if _, ok := db.keys[key]; !ok { + c.WriteInt(expiretimeReplyMissingKey) + return + } + + ttl, ok := db.ttl[key] + if !ok { + c.WriteInt(expiretimeReplyNoExpiration) + return + } + + c.WriteInt(timeResultStrategy(m.effectiveNow().Add(ttl))) + }) + } +} + +// TOUCH +func (m *Miniredis) cmdTouch(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + count := 0 + for _, key := range args { + if db.exists(key) { + count++ + } + } + c.WriteInt(count) + }) +} + +// TTL +func (m *Miniredis) cmdTTL(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if _, ok := db.keys[key]; !ok { + // No such key + c.WriteInt(-2) + return + } + + v, ok := db.ttl[key] + if !ok { + // no expire value + c.WriteInt(-1) + return + } + c.WriteInt(int(v.Seconds())) + }) +} + +// PTTL +func (m *Miniredis) cmdPTTL(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if _, ok := db.keys[key]; !ok { + // no such key + c.WriteInt(-2) + return + } + + v, ok := db.ttl[key] + if !ok { + // no expire value + c.WriteInt(-1) + return + } + c.WriteInt(int(v.Nanoseconds() / 1000000)) + }) +} + +// PERSIST +func (m *Miniredis) cmdPersist(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if _, ok := db.keys[key]; !ok { + // no such key + c.WriteInt(0) + return + } + + if _, ok := db.ttl[key]; !ok { + // no expire value + c.WriteInt(0) + return + } + delete(db.ttl, key) + db.incr(key) + c.WriteInt(1) + }) +} + +// DEL and UNLINK +func (m *Miniredis) cmdDel(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + count := 0 + for _, key := range args { + if db.exists(key) { + count++ + } + db.del(key, true) // delete expire + } + c.WriteInt(count) + }) +} + +// DUMP +func (m *Miniredis) cmdDump(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + keyType, exists := db.keys[key] + if !exists { + c.WriteNull() + } else if keyType != keyTypeString { + c.WriteError(msgWrongType) + } else { + c.WriteBulk(db.stringGet(key)) + } + }) +} + +// TYPE +func (m *Miniredis) cmdType(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[key] + if !ok { + c.WriteInline("none") + return + } + + c.WriteInline(t) + }) +} + +// EXISTS +func (m *Miniredis) cmdExists(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + found := 0 + for _, k := range args { + if db.exists(k) { + found++ + } + } + c.WriteInt(found) + }) +} + +// MOVE +func (m *Miniredis) cmdMove(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + var opts struct { + key string + targetDB int + } + + opts.key = args[0] + opts.targetDB, _ = strconv.Atoi(args[1]) + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + if ctx.selectedDB == opts.targetDB { + c.WriteError("ERR source and destination objects are the same") + return + } + db := m.db(ctx.selectedDB) + targetDB := m.db(opts.targetDB) + + if !db.move(opts.key, targetDB) { + c.WriteInt(0) + return + } + c.WriteInt(1) + }) +} + +// KEYS +func (m *Miniredis) cmdKeys(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + keys, _ := matchKeys(db.allKeys(), key) + c.WriteLen(len(keys)) + for _, s := range keys { + c.WriteBulk(s) + } + }) +} + +// RANDOMKEY +func (m *Miniredis) cmdRandomkey(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(0)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if len(db.keys) == 0 { + c.WriteNull() + return + } + nr := m.randIntn(len(db.keys)) + for k := range db.keys { + if nr == 0 { + c.WriteBulk(k) + return + } + nr-- + } + }) +} + +// RENAME +func (m *Miniredis) cmdRename(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + opts := struct { + from string + to string + }{ + from: args[0], + to: args[1], + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.from) { + c.WriteError(msgKeyNotFound) + return + } + + db.rename(opts.from, opts.to) + c.WriteOK() + }) +} + +// RENAMENX +func (m *Miniredis) cmdRenamenx(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + opts := struct { + from string + to string + }{ + from: args[0], + to: args[1], + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.from) { + c.WriteError(msgKeyNotFound) + return + } + + if db.exists(opts.to) { + c.WriteInt(0) + return + } + + db.rename(opts.from, opts.to) + c.WriteInt(1) + }) +} + +type restoreOpts struct { + key string + serializedValue string + rawTtl string + replace bool + absTtl bool +} + +func restoreParse(args []string) *restoreOpts { + var opts restoreOpts + + opts.key, opts.rawTtl, opts.serializedValue, args = args[0], args[1], args[2], args[3:] + + for len(args) > 0 { + switch arg := strings.ToUpper(args[0]); arg { + case "REPLACE": + opts.replace = true + case "ABSTTL": + opts.absTtl = true + default: + return nil + } + + args = args[1:] + } + + return &opts +} + +// RESTORE +func (m *Miniredis) cmdRestore(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + var opts = restoreParse(args) + if opts == nil { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + _, keyExists := db.keys[opts.key] + if keyExists && !opts.replace { + setDirty(c) + c.WriteError("BUSYKEY Target key name already exists.") + return + } + + ttl, err := strconv.Atoi(opts.rawTtl) + if err != nil || ttl < 0 { + c.WriteError(msgInvalidInt) + return + } + + db.stringSet(opts.key, opts.serializedValue) + + if ttl != 0 { + if opts.absTtl { + db.ttl[opts.key] = m.at(ttl, time.Millisecond) + } else { + db.ttl[opts.key] = time.Duration(ttl) * time.Millisecond + } + } + + c.WriteOK() + }) +} + +type scanOpts struct { + cursor int + count int + withMatch bool + match string + withType bool + _type string +} + +func scanParse(cmd string, args []string) (*scanOpts, error) { + var opts scanOpts + if err := optIntSimple(args[0], &opts.cursor); err != nil { + return nil, errors.New(msgInvalidCursor) + } + args = args[1:] + + // MATCH, COUNT and TYPE options + for len(args) > 0 { + if strings.ToLower(args[0]) == "count" { + if len(args) < 2 { + return nil, errors.New(msgSyntaxError) + } + count, err := strconv.Atoi(args[1]) + if err != nil || count < 0 { + return nil, errors.New(msgInvalidInt) + } + if count == 0 { + return nil, errors.New(msgSyntaxError) + } + opts.count = count + args = args[2:] + continue + } + if strings.ToLower(args[0]) == "match" { + if len(args) < 2 { + return nil, errors.New(msgSyntaxError) + } + opts.withMatch = true + opts.match, args = args[1], args[2:] + continue + } + if strings.ToLower(args[0]) == "type" { + if len(args) < 2 { + return nil, errors.New(msgSyntaxError) + } + opts.withType = true + opts._type, args = strings.ToLower(args[1]), args[2:] + continue + } + return nil, errors.New(msgSyntaxError) + } + return &opts, nil +} + +// SCAN +func (m *Miniredis) cmdScan(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + opts, err := scanParse(cmd, args) + if err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + // We return _all_ (matched) keys every time, so that cursors work. + // We ignore "COUNT", which is allowed according to the Redis docs. + var keys []string + + if opts.withType { + keys = make([]string, 0) + for k, t := range db.keys { + // type must be given exactly; no pattern matching is performed + if t == opts._type { + keys = append(keys, k) + } + } + } else { + keys = db.allKeys() + } + + sort.Strings(keys) // To make things deterministic. + + if opts.withMatch { + keys, _ = matchKeys(keys, opts.match) + } + + // we only ever return all at once, so no non-zero cursor can every be valid + if opts.cursor != 0 { + c.WriteLen(2) + c.WriteBulk("0") // no next cursor + c.WriteLen(0) // no elements + return + } + cursorValue := 0 // we don't use cursors + c.WriteLen(2) + c.WriteBulk(fmt.Sprintf("%d", cursorValue)) + c.WriteLen(len(keys)) + for _, k := range keys { + c.WriteBulk(k) + } + }) +} + +type copyOpts struct { + from string + to string + destinationDB int + replace bool +} + +func copyParse(cmd string, args []string) (*copyOpts, error) { + opts := copyOpts{ + destinationDB: -1, + } + + opts.from, opts.to, args = args[0], args[1], args[2:] + for len(args) > 0 { + switch strings.ToLower(args[0]) { + case "db": + if len(args) < 2 { + return nil, errors.New(msgSyntaxError) + } + if err := optIntSimple(args[1], &opts.destinationDB); err != nil { + return nil, err + } + if opts.destinationDB < 0 { + return nil, errors.New(msgDBIndexOutOfRange) + } + args = args[2:] + case "replace": + opts.replace = true + args = args[1:] + default: + return nil, errors.New(msgSyntaxError) + } + } + return &opts, nil +} + +// COPY +func (m *Miniredis) cmdCopy(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + opts, err := copyParse(cmd, args) + if err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + fromDB, toDB := ctx.selectedDB, opts.destinationDB + if toDB == -1 { + toDB = fromDB + } + + if fromDB == toDB && opts.from == opts.to { + c.WriteError("ERR source and destination objects are the same") + return + } + + if !m.db(fromDB).exists(opts.from) { + c.WriteInt(0) + return + } + + if !opts.replace { + if m.db(toDB).exists(opts.to) { + c.WriteInt(0) + return + } + } + + m.copy(m.db(fromDB), opts.from, m.db(toDB), opts.to) + c.WriteInt(1) + }) +} + +// WAIT +func (m *Miniredis) cmdWait(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + nReplicas, err := strconv.Atoi(args[0]) + if err != nil || nReplicas < 0 { + c.WriteError(msgInvalidInt) + return + } + timeout, err := strconv.Atoi(args[1]) + if err != nil { + c.WriteError(msgInvalidInt) + return + } + if timeout < 0 { + c.WriteError(msgTimeoutNegative) + return + } + // WAIT always returns 0 when called on a standalone instance + c.WriteInt(0) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_geo.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_geo.go new file mode 100644 index 00000000..12cf99ad --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_geo.go @@ -0,0 +1,577 @@ +// Commands from https://redis.io/commands#geo + +package miniredis + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsGeo handles GEOADD, GEORADIUS etc. +func commandsGeo(m *Miniredis) { + m.srv.Register("GEOADD", m.cmdGeoadd) + m.srv.Register("GEODIST", m.cmdGeodist, server.ReadOnlyOption()) + m.srv.Register("GEOPOS", m.cmdGeopos, server.ReadOnlyOption()) + m.srv.Register("GEORADIUS", m.cmdGeoradius) + m.srv.Register("GEORADIUS_RO", m.cmdGeoradius, server.ReadOnlyOption()) + m.srv.Register("GEORADIUSBYMEMBER", m.cmdGeoradiusbymember) + m.srv.Register("GEORADIUSBYMEMBER_RO", m.cmdGeoradiusbymember, server.ReadOnlyOption()) +} + +// GEOADD +func (m *Miniredis) cmdGeoadd(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + if len(args[1:])%3 != 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + + key, args := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if db.exists(key) && db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + toSet := map[string]float64{} + for len(args) > 2 { + rawLong, rawLat, name := args[0], args[1], args[2] + args = args[3:] + longitude, err := strconv.ParseFloat(rawLong, 64) + if err != nil { + c.WriteError("ERR value is not a valid float") + return + } + latitude, err := strconv.ParseFloat(rawLat, 64) + if err != nil { + c.WriteError("ERR value is not a valid float") + return + } + + if latitude < -85.05112878 || + latitude > 85.05112878 || + longitude < -180 || + longitude > 180 { + c.WriteError(fmt.Sprintf("ERR invalid longitude,latitude pair %.6f,%.6f", longitude, latitude)) + return + } + + toSet[name] = float64(toGeohash(longitude, latitude)) + } + + set := 0 + for name, score := range toSet { + if db.ssetAdd(key, score, name) { + set++ + } + } + c.WriteInt(set) + }) +} + +// GEODIST +func (m *Miniredis) cmdGeodist(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + key, from, to, args := args[0], args[1], args[2], args[3:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + if !db.exists(key) { + c.WriteNull() + return + } + if db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + unit := "m" + if len(args) > 0 { + unit, args = args[0], args[1:] + } + if len(args) > 0 { + c.WriteError(msgSyntaxError) + return + } + + toMeter := parseUnit(unit) + if toMeter == 0 { + c.WriteError(msgUnsupportedUnit) + return + } + + members := db.sortedsetKeys[key] + fromD, okFrom := members.get(from) + toD, okTo := members.get(to) + if !okFrom || !okTo { + c.WriteNull() + return + } + + fromLo, fromLat := fromGeohash(uint64(fromD)) + toLo, toLat := fromGeohash(uint64(toD)) + + dist := distance(fromLat, fromLo, toLat, toLo) / toMeter + c.WriteBulk(fmt.Sprintf("%.4f", dist)) + }) +} + +// GEOPOS +func (m *Miniredis) cmdGeopos(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + key, args := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if db.exists(key) && db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + c.WriteLen(len(args)) + for _, l := range args { + if !db.ssetExists(key, l) { + c.WriteLen(-1) + continue + } + score := db.ssetScore(key, l) + c.WriteLen(2) + long, lat := fromGeohash(uint64(score)) + c.WriteBulk(fmt.Sprintf("%f", long)) + c.WriteBulk(fmt.Sprintf("%f", lat)) + } + }) +} + +type geoDistance struct { + Name string + Score float64 + Distance float64 + Longitude float64 + Latitude float64 +} + +// GEORADIUS and GEORADIUS_RO +func (m *Miniredis) cmdGeoradius(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(5)) { + return + } + + key := args[0] + longitude, err := strconv.ParseFloat(args[1], 64) + if err != nil { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + latitude, err := strconv.ParseFloat(args[2], 64) + if err != nil { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + radius, err := strconv.ParseFloat(args[3], 64) + if err != nil || radius < 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + toMeter := parseUnit(args[4]) + if toMeter == 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + args = args[5:] + + var opts struct { + withDist bool + withCoord bool + direction direction // unsorted + count int + withStore bool + storeKey string + withStoredist bool + storedistKey string + } + for len(args) > 0 { + arg := args[0] + args = args[1:] + switch strings.ToUpper(arg) { + case "WITHCOORD": + opts.withCoord = true + case "WITHDIST": + opts.withDist = true + case "ASC": + opts.direction = asc + case "DESC": + opts.direction = desc + case "COUNT": + if len(args) == 0 { + setDirty(c) + c.WriteError("ERR syntax error") + return + } + n, err := strconv.Atoi(args[0]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if n <= 0 { + setDirty(c) + c.WriteError("ERR COUNT must be > 0") + return + } + args = args[1:] + opts.count = n + case "STORE": + if len(args) == 0 { + setDirty(c) + c.WriteError("ERR syntax error") + return + } + opts.withStore = true + opts.storeKey = args[0] + args = args[1:] + case "STOREDIST": + if len(args) == 0 { + setDirty(c) + c.WriteError("ERR syntax error") + return + } + opts.withStoredist = true + opts.storedistKey = args[0] + args = args[1:] + default: + setDirty(c) + c.WriteError("ERR syntax error") + return + } + } + + if strings.ToUpper(cmd) == "GEORADIUS_RO" && (opts.withStore || opts.withStoredist) { + setDirty(c) + c.WriteError("ERR syntax error") + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + if (opts.withStore || opts.withStoredist) && (opts.withDist || opts.withCoord) { + c.WriteError("ERR STORE option in GEORADIUS is not compatible with WITHDIST, WITHHASH and WITHCOORDS options") + return + } + + db := m.db(ctx.selectedDB) + members := db.ssetElements(key) + + matches := withinRadius(members, longitude, latitude, radius*toMeter) + + // deal with ASC/DESC + if opts.direction != unsorted { + sort.Slice(matches, func(i, j int) bool { + if opts.direction == desc { + return matches[i].Distance > matches[j].Distance + } + return matches[i].Distance < matches[j].Distance + }) + } + + // deal with COUNT + if opts.count > 0 && len(matches) > opts.count { + matches = matches[:opts.count] + } + + // deal with "STORE x" + if opts.withStore { + db.del(opts.storeKey, true) + for _, member := range matches { + db.ssetAdd(opts.storeKey, member.Score, member.Name) + } + c.WriteInt(len(matches)) + return + } + + // deal with "STOREDIST x" + if opts.withStoredist { + db.del(opts.storedistKey, true) + for _, member := range matches { + db.ssetAdd(opts.storedistKey, member.Distance/toMeter, member.Name) + } + c.WriteInt(len(matches)) + return + } + + c.WriteLen(len(matches)) + for _, member := range matches { + if !opts.withDist && !opts.withCoord { + c.WriteBulk(member.Name) + continue + } + + len := 1 + if opts.withDist { + len++ + } + if opts.withCoord { + len++ + } + c.WriteLen(len) + c.WriteBulk(member.Name) + if opts.withDist { + c.WriteBulk(fmt.Sprintf("%.4f", member.Distance/toMeter)) + } + if opts.withCoord { + c.WriteLen(2) + c.WriteBulk(fmt.Sprintf("%f", member.Longitude)) + c.WriteBulk(fmt.Sprintf("%f", member.Latitude)) + } + } + }) +} + +// GEORADIUSBYMEMBER and GEORADIUSBYMEMBER_RO +func (m *Miniredis) cmdGeoradiusbymember(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(4)) { + return + } + + opts := struct { + key string + member string + radius float64 + toMeter float64 + + withDist bool + withCoord bool + direction direction // unsorted + count int + withStore bool + storeKey string + withStoredist bool + storedistKey string + }{ + key: args[0], + member: args[1], + } + + r, err := strconv.ParseFloat(args[2], 64) + if err != nil || r < 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + opts.radius = r + + opts.toMeter = parseUnit(args[3]) + if opts.toMeter == 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + args = args[4:] + + for len(args) > 0 { + arg := args[0] + args = args[1:] + switch strings.ToUpper(arg) { + case "WITHCOORD": + opts.withCoord = true + case "WITHDIST": + opts.withDist = true + case "ASC": + opts.direction = asc + case "DESC": + opts.direction = desc + case "COUNT": + if len(args) == 0 { + setDirty(c) + c.WriteError("ERR syntax error") + return + } + n, err := strconv.Atoi(args[0]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if n <= 0 { + setDirty(c) + c.WriteError("ERR COUNT must be > 0") + return + } + args = args[1:] + opts.count = n + case "STORE": + if len(args) == 0 { + setDirty(c) + c.WriteError("ERR syntax error") + return + } + opts.withStore = true + opts.storeKey = args[0] + args = args[1:] + case "STOREDIST": + if len(args) == 0 { + setDirty(c) + c.WriteError("ERR syntax error") + return + } + opts.withStoredist = true + opts.storedistKey = args[0] + args = args[1:] + default: + setDirty(c) + c.WriteError("ERR syntax error") + return + } + } + + if strings.ToUpper(cmd) == "GEORADIUSBYMEMBER_RO" && (opts.withStore || opts.withStoredist) { + setDirty(c) + c.WriteError("ERR syntax error") + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + if (opts.withStore || opts.withStoredist) && (opts.withDist || opts.withCoord) { + c.WriteError("ERR STORE option in GEORADIUS is not compatible with WITHDIST, WITHHASH and WITHCOORDS options") + return + } + + db := m.db(ctx.selectedDB) + if !db.exists(opts.key) { + c.WriteNull() + return + } + + if db.t(opts.key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + // get position of member + if !db.ssetExists(opts.key, opts.member) { + c.WriteError("ERR could not decode requested zset member") + return + } + score := db.ssetScore(opts.key, opts.member) + longitude, latitude := fromGeohash(uint64(score)) + + members := db.ssetElements(opts.key) + matches := withinRadius(members, longitude, latitude, opts.radius*opts.toMeter) + + // deal with ASC/DESC + if opts.direction != unsorted { + sort.Slice(matches, func(i, j int) bool { + if opts.direction == desc { + return matches[i].Distance > matches[j].Distance + } + return matches[i].Distance < matches[j].Distance + }) + } + + // deal with COUNT + if opts.count > 0 && len(matches) > opts.count { + matches = matches[:opts.count] + } + + // deal with "STORE x" + if opts.withStore { + db.del(opts.storeKey, true) + for _, member := range matches { + db.ssetAdd(opts.storeKey, member.Score, member.Name) + } + c.WriteInt(len(matches)) + return + } + + // deal with "STOREDIST x" + if opts.withStoredist { + db.del(opts.storedistKey, true) + for _, member := range matches { + db.ssetAdd(opts.storedistKey, member.Distance/opts.toMeter, member.Name) + } + c.WriteInt(len(matches)) + return + } + + c.WriteLen(len(matches)) + for _, member := range matches { + if !opts.withDist && !opts.withCoord { + c.WriteBulk(member.Name) + continue + } + + len := 1 + if opts.withDist { + len++ + } + if opts.withCoord { + len++ + } + c.WriteLen(len) + c.WriteBulk(member.Name) + if opts.withDist { + c.WriteBulk(fmt.Sprintf("%.4f", member.Distance/opts.toMeter)) + } + if opts.withCoord { + c.WriteLen(2) + c.WriteBulk(fmt.Sprintf("%f", member.Longitude)) + c.WriteBulk(fmt.Sprintf("%f", member.Latitude)) + } + } + }) +} + +func withinRadius(members []ssElem, longitude, latitude, radius float64) []geoDistance { + matches := []geoDistance{} + for _, el := range members { + elLo, elLat := fromGeohash(uint64(el.score)) + distanceInMeter := distance(latitude, longitude, elLat, elLo) + + if distanceInMeter <= radius { + matches = append(matches, geoDistance{ + Name: el.member, + Score: el.score, + Distance: distanceInMeter, + Longitude: elLo, + Latitude: elLat, + }) + } + } + return matches +} + +func parseUnit(u string) float64 { + switch strings.ToLower(u) { + case "m": + return 1 + case "km": + return 1000 + case "mi": + return 1609.34 + case "ft": + return 0.3048 + default: + return 0 + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_hash.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_hash.go new file mode 100644 index 00000000..9c5e6866 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_hash.go @@ -0,0 +1,797 @@ +// Commands from https://redis.io/commands#hash + +package miniredis + +import ( + "fmt" + "math/big" + "strconv" + "strings" + "time" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsHash handles all hash value operations. +func commandsHash(m *Miniredis) { + m.srv.Register("HDEL", m.cmdHdel) + m.srv.Register("HEXISTS", m.cmdHexists, server.ReadOnlyOption()) + m.srv.Register("HGET", m.cmdHget, server.ReadOnlyOption()) + m.srv.Register("HGETALL", m.cmdHgetall, server.ReadOnlyOption()) + m.srv.Register("HINCRBY", m.cmdHincrby) + m.srv.Register("HINCRBYFLOAT", m.cmdHincrbyfloat) + m.srv.Register("HKEYS", m.cmdHkeys, server.ReadOnlyOption()) + m.srv.Register("HLEN", m.cmdHlen, server.ReadOnlyOption()) + m.srv.Register("HMGET", m.cmdHmget, server.ReadOnlyOption()) + m.srv.Register("HMSET", m.cmdHmset) + m.srv.Register("HSET", m.cmdHset) + m.srv.Register("HSETNX", m.cmdHsetnx) + m.srv.Register("HSTRLEN", m.cmdHstrlen, server.ReadOnlyOption()) + m.srv.Register("HVALS", m.cmdHvals, server.ReadOnlyOption()) + m.srv.Register("HSCAN", m.cmdHscan, server.ReadOnlyOption()) + m.srv.Register("HRANDFIELD", m.cmdHrandfield, server.ReadOnlyOption()) + m.srv.Register("HEXPIRE", m.cmdHexpire) +} + +// HSET +func (m *Miniredis) cmdHset(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + key, pairs := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if len(pairs)%2 == 1 { + c.WriteError(errWrongNumber(cmd)) + return + } + + if t, ok := db.keys[key]; ok && t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + new := db.hashSet(key, pairs...) + c.WriteInt(new) + }) +} + +// HSETNX +func (m *Miniredis) cmdHsetnx(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + opts := struct { + key string + field string + value string + }{ + key: args[0], + field: args[1], + value: args[2], + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + if _, ok := db.hashKeys[opts.key]; !ok { + db.hashKeys[opts.key] = map[string]string{} + db.keys[opts.key] = keyTypeHash + } + _, ok := db.hashKeys[opts.key][opts.field] + if ok { + c.WriteInt(0) + return + } + db.hashKeys[opts.key][opts.field] = opts.value + db.incr(opts.key) + c.WriteInt(1) + }) +} + +// HMSET +func (m *Miniredis) cmdHmset(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + key, args := args[0], args[1:] + if len(args)%2 != 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[key]; ok && t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + for len(args) > 0 { + field, value := args[0], args[1] + args = args[2:] + db.hashSet(key, field, value) + } + c.WriteOK() + }) +} + +// HGET +func (m *Miniredis) cmdHget(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + key, field := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[key] + if !ok { + c.WriteNull() + return + } + if t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + value, ok := db.hashKeys[key][field] + if !ok { + c.WriteNull() + return + } + c.WriteBulk(value) + }) +} + +// HDEL +func (m *Miniredis) cmdHdel(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + opts := struct { + key string + fields []string + }{ + key: args[0], + fields: args[1:], + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[opts.key] + if !ok { + // No key is zero deleted + c.WriteInt(0) + return + } + if t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + deleted := 0 + for _, f := range opts.fields { + _, ok := db.hashKeys[opts.key][f] + if !ok { + continue + } + delete(db.hashKeys[opts.key], f) + deleted++ + } + c.WriteInt(deleted) + + // Nothing left. Remove the whole key. + if len(db.hashKeys[opts.key]) == 0 { + db.del(opts.key, true) + } + }) +} + +// HEXISTS +func (m *Miniredis) cmdHexists(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + opts := struct { + key string + field string + }{ + key: args[0], + field: args[1], + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[opts.key] + if !ok { + c.WriteInt(0) + return + } + if t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + if _, ok := db.hashKeys[opts.key][opts.field]; !ok { + c.WriteInt(0) + return + } + c.WriteInt(1) + }) +} + +// HGETALL +func (m *Miniredis) cmdHgetall(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[key] + if !ok { + c.WriteMapLen(0) + return + } + if t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + c.WriteMapLen(len(db.hashKeys[key])) + for _, k := range db.hashFields(key) { + c.WriteBulk(k) + c.WriteBulk(db.hashGet(key, k)) + } + }) +} + +// HKEYS +func (m *Miniredis) cmdHkeys(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteLen(0) + return + } + if db.t(key) != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + fields := db.hashFields(key) + c.WriteLen(len(fields)) + for _, f := range fields { + c.WriteBulk(f) + } + }) +} + +// HSTRLEN +func (m *Miniredis) cmdHstrlen(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + hash, key := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[hash] + if !ok { + c.WriteInt(0) + return + } + if t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + keys := db.hashKeys[hash] + c.WriteInt(len(keys[key])) + }) +} + +// HVALS +func (m *Miniredis) cmdHvals(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[key] + if !ok { + c.WriteLen(0) + return + } + if t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + vals := db.hashValues(key) + c.WriteLen(len(vals)) + for _, v := range vals { + c.WriteBulk(v) + } + }) +} + +// HLEN +func (m *Miniredis) cmdHlen(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[key] + if !ok { + c.WriteInt(0) + return + } + if t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + c.WriteInt(len(db.hashKeys[key])) + }) +} + +// HMGET +func (m *Miniredis) cmdHmget(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[key]; ok && t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + f, ok := db.hashKeys[key] + if !ok { + f = map[string]string{} + } + + c.WriteLen(len(args) - 1) + for _, k := range args[1:] { + v, ok := f[k] + if !ok { + c.WriteNull() + continue + } + c.WriteBulk(v) + } + }) +} + +// HINCRBY +func (m *Miniredis) cmdHincrby(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + opts := struct { + key string + field string + delta int + }{ + key: args[0], + field: args[1], + } + if ok := optInt(c, args[2], &opts.delta); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + v, err := db.hashIncr(opts.key, opts.field, opts.delta) + if err != nil { + c.WriteError(err.Error()) + return + } + c.WriteInt(v) + }) +} + +// HINCRBYFLOAT +func (m *Miniredis) cmdHincrbyfloat(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + opts := struct { + key string + field string + delta *big.Float + }{ + key: args[0], + field: args[1], + } + delta, _, err := big.ParseFloat(args[2], 10, 128, 0) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidFloat) + return + } + opts.delta = delta + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + v, err := db.hashIncrfloat(opts.key, opts.field, opts.delta) + if err != nil { + c.WriteError(err.Error()) + return + } + c.WriteBulk(formatBig(v)) + }) +} + +// HSCAN +func (m *Miniredis) cmdHscan(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + opts := struct { + key string + cursor int + withMatch bool + match string + }{ + key: args[0], + } + if ok := optIntErr(c, args[1], &opts.cursor, msgInvalidCursor); !ok { + return + } + args = args[2:] + + // MATCH and COUNT options + for len(args) > 0 { + if strings.ToLower(args[0]) == "count" { + // we do nothing with count + if len(args) < 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + _, err := strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + args = args[2:] + continue + } + if strings.ToLower(args[0]) == "match" { + if len(args) < 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + opts.withMatch = true + opts.match, args = args[1], args[2:] + continue + } + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + // return _all_ (matched) keys every time + + if opts.cursor != 0 { + // Invalid cursor. + c.WriteLen(2) + c.WriteBulk("0") // no next cursor + c.WriteLen(0) // no elements + return + } + if db.exists(opts.key) && db.t(opts.key) != keyTypeHash { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.hashFields(opts.key) + if opts.withMatch { + members, _ = matchKeys(members, opts.match) + } + + c.WriteLen(2) + c.WriteBulk("0") // no next cursor + // HSCAN gives key, values. + c.WriteLen(len(members) * 2) + for _, k := range members { + c.WriteBulk(k) + c.WriteBulk(db.hashGet(opts.key, k)) + } + }) +} + +// HRANDFIELD +func (m *Miniredis) cmdHrandfield(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, between(1, 3)) { + return + } + + opts := struct { + key string + count int + countSet bool + withValues bool + }{ + key: args[0], + } + + if len(args) > 1 { + if ok := optIntErr(c, args[1], &opts.count, msgInvalidInt); !ok { + return + } + opts.countSet = true + } + + if len(args) == 3 { + if strings.ToLower(args[2]) == "withvalues" { + opts.withValues = true + } else { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + } + + withTx(m, c, func(peer *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + members := db.hashFields(opts.key) + m.shuffle(members) + + if !opts.countSet { + // > When called with just the key argument, return a random field from the + // hash value stored at key. + if len(members) == 0 { + peer.WriteNull() + return + } + peer.WriteBulk(members[0]) + return + } + + if len(members) > abs(opts.count) { + members = members[:abs(opts.count)] + } + switch { + case opts.count >= 0: + // if count is positive there can't be duplicates, and the length is restricted + case opts.count < 0: + // if count is negative there can be duplicates, but length will match + if len(members) > 0 { + for len(members) < -opts.count { + members = append(members, members[m.randIntn(len(members))]) + } + } + } + + if opts.withValues { + peer.WriteMapLen(len(members)) + for _, m := range members { + peer.WriteBulk(m) + peer.WriteBulk(db.hashGet(opts.key, m)) + } + return + } + peer.WriteLen(len(members)) + for _, m := range members { + peer.WriteBulk(m) + } + }) +} + +// HEXPIRE +func (m *Miniredis) cmdHexpire(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(5)) { + return + } + + opts, err := parseHExpireArgs(args) + if err != "" { + setDirty(c) + c.WriteError(err) + return + } + + withTx(m, c, func(peer *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if _, ok := db.keys[opts.key]; !ok { + c.WriteLen(len(opts.fields)) + for range opts.fields { + c.WriteInt(-2) + } + return + } + + if db.t(opts.key) != keyTypeHash { + c.WriteError(msgWrongType) + return + } + + fieldTTLs := db.hashTTLs[opts.key] + if fieldTTLs == nil { + fieldTTLs = map[string]time.Duration{} + db.hashTTLs[opts.key] = fieldTTLs + } + + c.WriteLen(len(opts.fields)) + for _, field := range opts.fields { + if _, ok := db.hashKeys[opts.key][field]; !ok { + c.WriteInt(-2) + continue + } + + currentTtl, ok := fieldTTLs[field] + newTTL := time.Duration(opts.ttl) * time.Second + + // NX -- For each specified field, + // set expiration only when the field has no expiration. + if opts.nx && ok { + c.WriteInt(0) + continue + } + + // XX -- For each specified field, + // set expiration only when the field has an existing expiration. + if opts.xx && !ok { + c.WriteInt(0) + continue + } + + // GT -- For each specified field, + // set expiration only when the new expiration is greater than current one. + if opts.gt && (!ok || newTTL <= currentTtl) { + c.WriteInt(0) + continue + } + + // LT -- For each specified field, + // set expiration only when the new expiration is less than current one. + if opts.lt && ok && newTTL >= currentTtl { + c.WriteInt(0) + continue + } + + fieldTTLs[field] = newTTL + c.WriteInt(1) + } + }) +} + +type hexpireOpts struct { + key string + ttl int + nx bool + xx bool + gt bool + lt bool + fields []string +} + +func parseHExpireArgs(args []string) (hexpireOpts, string) { + var opts hexpireOpts + opts.key = args[0] + + if err := optIntSimple(args[1], &opts.ttl); err != nil { + return hexpireOpts{}, err.Error() + } + + args = args[2:] + + for len(args) > 0 { + switch strings.ToLower(args[0]) { + case "nx": + opts.nx = true + args = args[1:] + case "xx": + opts.xx = true + args = args[1:] + case "gt": + opts.gt = true + args = args[1:] + case "lt": + opts.lt = true + args = args[1:] + case "fields": + var numFields int + if err := optIntSimple(args[1], &numFields); err != nil { + return hexpireOpts{}, msgNumFieldsInvalid + } + if numFields <= 0 { + return hexpireOpts{}, msgNumFieldsInvalid + } + + // FIELDS numFields field1 field2 ... + if len(args) < 2+numFields { + return hexpireOpts{}, msgNumFieldsParameter + } + + opts.fields = append([]string{}, args[2:2+numFields]...) + args = args[2+numFields:] + default: + return hexpireOpts{}, fmt.Sprintf(msgMandatoryArgument, "FIELDS") + } + } + + if opts.gt && opts.lt { + return hexpireOpts{}, msgGTandLT + } + + if opts.nx && (opts.xx || opts.gt || opts.lt) { + return hexpireOpts{}, msgNXandXXGTLT + } + + return opts, "" +} + +func abs(n int) int { + if n < 0 { + return -n + } + return n +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_hll.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_hll.go new file mode 100644 index 00000000..7bfc9504 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_hll.go @@ -0,0 +1,71 @@ +package miniredis + +import "github.com/alicebob/miniredis/v2/server" + +// commandsHll handles all hll related operations. +func commandsHll(m *Miniredis) { + m.srv.Register("PFADD", m.cmdPfadd) + m.srv.Register("PFCOUNT", m.cmdPfcount, server.ReadOnlyOption()) + m.srv.Register("PFMERGE", m.cmdPfmerge) +} + +// PFADD +func (m *Miniredis) cmdPfadd(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, items := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if db.exists(key) && db.t(key) != keyTypeHll { + c.WriteError(ErrNotValidHllValue.Error()) + return + } + + altered := db.hllAdd(key, items...) + c.WriteInt(altered) + }) +} + +// PFCOUNT +func (m *Miniredis) cmdPfcount(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + keys := args + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + count, err := db.hllCount(keys) + if err != nil { + c.WriteError(err.Error()) + return + } + + c.WriteInt(count) + }) +} + +// PFMERGE +func (m *Miniredis) cmdPfmerge(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + keys := args + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if err := db.hllMerge(keys); err != nil { + c.WriteError(err.Error()) + return + } + c.WriteOK() + }) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_info.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_info.go new file mode 100644 index 00000000..9fa84711 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_info.go @@ -0,0 +1,43 @@ +package miniredis + +import ( + "fmt" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +const ( + clientsSectionName = "clients" + clientsSectionContent = "# Clients\nconnected_clients:%d\r\n" + + statsSectionName = "stats" + statsSectionContent = "# Stats\ntotal_connections_received:%d\r\ntotal_commands_processed:%d\r\n" +) + +// Command 'INFO' from https://redis.io/commands/info/ +func (m *Miniredis) cmdInfo(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, between(0, 1)) { + return + } + var result string + if len(args) == 0 { + result = fmt.Sprintf(clientsSectionContent, m.Server().ClientsLen()) + fmt.Sprintf(statsSectionContent, m.Server().TotalConnections(), m.Server().TotalCommands()) + c.WriteBulk(result) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + switch section := strings.ToLower(args[0]); section { + case clientsSectionName: + result = fmt.Sprintf(clientsSectionContent, m.Server().ClientsLen()) + case statsSectionName: + result = fmt.Sprintf(statsSectionContent, m.Server().TotalConnections(), m.Server().TotalCommands()) + default: + setDirty(c) + c.WriteError(fmt.Sprintf("section (%s) is not supported", section)) + return + } + c.WriteBulk(result) + }) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_list.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_list.go new file mode 100644 index 00000000..9de16359 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_list.go @@ -0,0 +1,931 @@ +// Commands from https://redis.io/commands#list + +package miniredis + +import ( + "strconv" + "strings" + "time" + + "github.com/alicebob/miniredis/v2/server" +) + +type leftright int + +const ( + left leftright = iota + right +) + +// commandsList handles list commands (mostly L*) +func commandsList(m *Miniredis) { + m.srv.Register("BLPOP", m.cmdBlpop) + m.srv.Register("BRPOP", m.cmdBrpop) + m.srv.Register("BRPOPLPUSH", m.cmdBrpoplpush) + m.srv.Register("LINDEX", m.cmdLindex, server.ReadOnlyOption()) + m.srv.Register("LPOS", m.cmdLpos, server.ReadOnlyOption()) + m.srv.Register("LINSERT", m.cmdLinsert) + m.srv.Register("LLEN", m.cmdLlen, server.ReadOnlyOption()) + m.srv.Register("LPOP", m.cmdLpop) + m.srv.Register("LPUSH", m.cmdLpush) + m.srv.Register("LPUSHX", m.cmdLpushx) + m.srv.Register("LRANGE", m.cmdLrange, server.ReadOnlyOption()) + m.srv.Register("LREM", m.cmdLrem) + m.srv.Register("LSET", m.cmdLset) + m.srv.Register("LTRIM", m.cmdLtrim) + m.srv.Register("RPOP", m.cmdRpop) + m.srv.Register("RPOPLPUSH", m.cmdRpoplpush) + m.srv.Register("RPUSH", m.cmdRpush) + m.srv.Register("RPUSHX", m.cmdRpushx) + m.srv.Register("LMOVE", m.cmdLmove) + m.srv.Register("BLMOVE", m.cmdBlmove) +} + +// BLPOP +func (m *Miniredis) cmdBlpop(c *server.Peer, cmd string, args []string) { + m.cmdBXpop(c, cmd, args, left) +} + +// BRPOP +func (m *Miniredis) cmdBrpop(c *server.Peer, cmd string, args []string) { + m.cmdBXpop(c, cmd, args, right) +} + +func (m *Miniredis) cmdBXpop(c *server.Peer, cmd string, args []string, lr leftright) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + var opts struct { + keys []string + timeout time.Duration + } + + if ok := optDuration(c, args[len(args)-1], &opts.timeout); !ok { + return + } + opts.keys = args[:len(args)-1] + + blocking( + m, + c, + opts.timeout, + func(c *server.Peer, ctx *connCtx) bool { + db := m.db(ctx.selectedDB) + for _, key := range opts.keys { + if !db.exists(key) { + continue + } + if db.t(key) != keyTypeList { + c.WriteError(msgWrongType) + return true + } + + if len(db.listKeys[key]) == 0 { + continue + } + c.WriteLen(2) + c.WriteBulk(key) + var v string + switch lr { + case left: + v = db.listLpop(key) + case right: + v = db.listPop(key) + } + c.WriteBulk(v) + return true + } + return false + }, + func(c *server.Peer) { + // timeout + c.WriteLen(-1) + }, + ) +} + +// LINDEX +func (m *Miniredis) cmdLindex(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + key, offsets := args[0], args[1] + + offset, err := strconv.Atoi(offsets) + if err != nil || offsets == "-0" { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[key] + if !ok { + // No such key + c.WriteNull() + return + } + if t != keyTypeList { + c.WriteError(msgWrongType) + return + } + + l := db.listKeys[key] + if offset < 0 { + offset = len(l) + offset + } + if offset < 0 || offset > len(l)-1 { + c.WriteNull() + return + } + c.WriteBulk(l[offset]) + }) +} + +// LPOS key element [RANK rank] [COUNT num-matches] [MAXLEN len] +func (m *Miniredis) cmdLpos(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + // Extract options from arguments if present. + // + // Redis allows duplicate options and uses the last specified. + // `LPOS key term RANK 1 RANK 2` is effectively the same as + // `LPOS key term RANK 2` + if len(args)%2 == 1 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + rank, count := 1, 1 // Default values + var maxlen int // Default value is the list length (see below) + var countSpecified, maxlenSpecified bool + if len(args) > 2 { + for i := 2; i < len(args); i++ { + if i%2 == 0 { + val := args[i+1] + var err error + switch strings.ToLower(args[i]) { + case "rank": + if rank, err = strconv.Atoi(val); err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if rank == 0 { + setDirty(c) + c.WriteError(msgRankIsZero) + return + } + case "count": + countSpecified = true + if count, err = strconv.Atoi(val); err != nil || count < 0 { + setDirty(c) + c.WriteError(msgCountIsNegative) + return + } + case "maxlen": + maxlenSpecified = true + if maxlen, err = strconv.Atoi(val); err != nil || maxlen < 0 { + setDirty(c) + c.WriteError(msgMaxLengthIsNegative) + return + } + default: + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + } + } + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + key, element := args[0], args[1] + t, ok := db.keys[key] + if !ok { + // No such key + c.WriteNull() + return + } + if t != keyTypeList { + c.WriteError(msgWrongType) + return + } + l := db.listKeys[key] + + // RANK cannot be zero (see above). + // If RANK is positive search forward (left to right). + // If RANK is negative search backward (right to left). + // Iterator returns true to continue iterating. + iterate := func(iterator func(i int, e string) bool) { + comparisons := len(l) + // Only use max length if specified, not zero, and less than total length. + // When max length is specified, but is zero, this means "unlimited". + if maxlenSpecified && maxlen != 0 && maxlen < len(l) { + comparisons = maxlen + } + if rank > 0 { + for i := 0; i < comparisons; i++ { + if resume := iterator(i, l[i]); !resume { + return + } + } + } else if rank < 0 { + start := len(l) - 1 + end := len(l) - comparisons + for i := start; i >= end; i-- { + if resume := iterator(i, l[i]); !resume { + return + } + } + } + } + + var currentRank, currentCount int + vals := make([]int, 0, count) + iterate(func(i int, e string) bool { + if e == element { + currentRank++ + // Only collect values only after surpassing the absolute value of rank. + if rank > 0 && currentRank < rank { + return true + } + if rank < 0 && currentRank < -rank { + return true + } + vals = append(vals, i) + currentCount++ + if currentCount == count { + return false + } + } + return true + }) + + if !countSpecified && len(vals) == 0 { + c.WriteNull() + return + } + if !countSpecified && len(vals) == 1 { + c.WriteInt(vals[0]) + return + } + c.WriteLen(len(vals)) + for _, val := range vals { + c.WriteInt(val) + } + }) +} + +// LINSERT +func (m *Miniredis) cmdLinsert(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(4)) { + return + } + + key := args[0] + where := 0 + switch strings.ToLower(args[1]) { + case "before": + where = -1 + case "after": + where = +1 + default: + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + pivot := args[2] + value := args[3] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[key] + if !ok { + // No such key + c.WriteInt(0) + return + } + if t != keyTypeList { + c.WriteError(msgWrongType) + return + } + + l := db.listKeys[key] + for i, el := range l { + if el != pivot { + continue + } + + if where < 0 { + l = append(l[:i], append(listKey{value}, l[i:]...)...) + } else { + if i == len(l)-1 { + l = append(l, value) + } else { + l = append(l[:i+1], append(listKey{value}, l[i+1:]...)...) + } + } + db.listKeys[key] = l + db.incr(key) + c.WriteInt(len(l)) + return + } + c.WriteInt(-1) + }) +} + +// LLEN +func (m *Miniredis) cmdLlen(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[key] + if !ok { + // No such key. That's zero length. + c.WriteInt(0) + return + } + if t != keyTypeList { + c.WriteError(msgWrongType) + return + } + + c.WriteInt(len(db.listKeys[key])) + }) +} + +// LPOP +func (m *Miniredis) cmdLpop(c *server.Peer, cmd string, args []string) { + m.cmdXpop(c, cmd, args, left) +} + +// RPOP +func (m *Miniredis) cmdRpop(c *server.Peer, cmd string, args []string) { + m.cmdXpop(c, cmd, args, right) +} + +func (m *Miniredis) cmdXpop(c *server.Peer, cmd string, args []string, lr leftright) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + var opts struct { + key string + withCount bool + count int + } + + opts.key, args = args[0], args[1:] + if len(args) > 0 { + if ok := optInt(c, args[0], &opts.count); !ok { + return + } + if opts.count < 0 { + setDirty(c) + c.WriteError(msgOutOfRange) + return + } + opts.withCount = true + args = args[1:] + } + if len(args) > 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + // non-existing key is fine + if opts.withCount && !c.Resp3 { + // zero-length list in this specific case. Looks like a redis bug to me. + c.WriteLen(-1) + return + } + c.WriteNull() + return + } + if db.t(opts.key) != keyTypeList { + c.WriteError(msgWrongType) + return + } + + if opts.withCount { + var popped []string + for opts.count > 0 && len(db.listKeys[opts.key]) > 0 { + switch lr { + case left: + popped = append(popped, db.listLpop(opts.key)) + case right: + popped = append(popped, db.listPop(opts.key)) + } + opts.count -= 1 + } + c.WriteStrings(popped) + return + } + + var elem string + switch lr { + case left: + elem = db.listLpop(opts.key) + case right: + elem = db.listPop(opts.key) + } + c.WriteBulk(elem) + }) +} + +// LPUSH +func (m *Miniredis) cmdLpush(c *server.Peer, cmd string, args []string) { + m.cmdXpush(c, cmd, args, left) +} + +// RPUSH +func (m *Miniredis) cmdRpush(c *server.Peer, cmd string, args []string) { + m.cmdXpush(c, cmd, args, right) +} + +func (m *Miniredis) cmdXpush(c *server.Peer, cmd string, args []string, lr leftright) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, args := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if db.exists(key) && db.t(key) != keyTypeList { + c.WriteError(msgWrongType) + return + } + + var newLen int + for _, value := range args { + switch lr { + case left: + newLen = db.listLpush(key, value) + case right: + newLen = db.listPush(key, value) + } + } + c.WriteInt(newLen) + }) +} + +// LPUSHX +func (m *Miniredis) cmdLpushx(c *server.Peer, cmd string, args []string) { + m.cmdXpushx(c, cmd, args, left) +} + +// RPUSHX +func (m *Miniredis) cmdRpushx(c *server.Peer, cmd string, args []string) { + m.cmdXpushx(c, cmd, args, right) +} + +func (m *Miniredis) cmdXpushx(c *server.Peer, cmd string, args []string, lr leftright) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, args := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteInt(0) + return + } + if db.t(key) != keyTypeList { + c.WriteError(msgWrongType) + return + } + + var newLen int + for _, value := range args { + switch lr { + case left: + newLen = db.listLpush(key, value) + case right: + newLen = db.listPush(key, value) + } + } + c.WriteInt(newLen) + }) +} + +// LRANGE +func (m *Miniredis) cmdLrange(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + opts := struct { + key string + start int + end int + }{ + key: args[0], + } + if ok := optInt(c, args[1], &opts.start); !ok { + return + } + if ok := optInt(c, args[2], &opts.end); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeList { + c.WriteError(msgWrongType) + return + } + + l := db.listKeys[opts.key] + if len(l) == 0 { + c.WriteLen(0) + return + } + + rs, re := redisRange(len(l), opts.start, opts.end, false) + c.WriteLen(re - rs) + for _, el := range l[rs:re] { + c.WriteBulk(el) + } + }) +} + +// LREM +func (m *Miniredis) cmdLrem(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + count int + value string + } + opts.key = args[0] + if ok := optInt(c, args[1], &opts.count); !ok { + return + } + opts.value = args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + c.WriteInt(0) + return + } + if db.t(opts.key) != keyTypeList { + c.WriteError(msgWrongType) + return + } + + l := db.listKeys[opts.key] + if opts.count < 0 { + reverseSlice(l) + } + deleted := 0 + newL := []string{} + toDelete := len(l) + if opts.count < 0 { + toDelete = -opts.count + } + if opts.count > 0 { + toDelete = opts.count + } + for _, el := range l { + if el == opts.value { + if toDelete > 0 { + deleted++ + toDelete-- + continue + } + } + newL = append(newL, el) + } + if opts.count < 0 { + reverseSlice(newL) + } + if len(newL) == 0 { + db.del(opts.key, true) + } else { + db.listKeys[opts.key] = newL + db.incr(opts.key) + } + + c.WriteInt(deleted) + }) +} + +// LSET +func (m *Miniredis) cmdLset(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + index int + value string + } + opts.key = args[0] + if ok := optInt(c, args[1], &opts.index); !ok { + return + } + opts.value = args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + c.WriteError(msgKeyNotFound) + return + } + if db.t(opts.key) != keyTypeList { + c.WriteError(msgWrongType) + return + } + + l := db.listKeys[opts.key] + index := opts.index + if index < 0 { + index = len(l) + index + } + if index < 0 || index > len(l)-1 { + c.WriteError(msgOutOfRange) + return + } + l[index] = opts.value + db.incr(opts.key) + + c.WriteOK() + }) +} + +// LTRIM +func (m *Miniredis) cmdLtrim(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + start int + end int + } + + opts.key = args[0] + if ok := optInt(c, args[1], &opts.start); !ok { + return + } + if ok := optInt(c, args[2], &opts.end); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.keys[opts.key] + if !ok { + c.WriteOK() + return + } + if t != keyTypeList { + c.WriteError(msgWrongType) + return + } + + l := db.listKeys[opts.key] + rs, re := redisRange(len(l), opts.start, opts.end, false) + l = l[rs:re] + if len(l) == 0 { + db.del(opts.key, true) + } else { + db.listKeys[opts.key] = l + db.incr(opts.key) + } + c.WriteOK() + }) +} + +// RPOPLPUSH +func (m *Miniredis) cmdRpoplpush(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + src, dst := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(src) { + c.WriteNull() + return + } + if db.t(src) != keyTypeList || (db.exists(dst) && db.t(dst) != keyTypeList) { + c.WriteError(msgWrongType) + return + } + elem := db.listPop(src) + db.listLpush(dst, elem) + c.WriteBulk(elem) + }) +} + +// BRPOPLPUSH +func (m *Miniredis) cmdBrpoplpush(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + src string + dst string + timeout time.Duration + } + opts.src = args[0] + opts.dst = args[1] + if ok := optDuration(c, args[2], &opts.timeout); !ok { + return + } + + blocking( + m, + c, + opts.timeout, + func(c *server.Peer, ctx *connCtx) bool { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.src) { + return false + } + if db.t(opts.src) != keyTypeList || (db.exists(opts.dst) && db.t(opts.dst) != keyTypeList) { + c.WriteError(msgWrongType) + return true + } + if len(db.listKeys[opts.src]) == 0 { + return false + } + elem := db.listPop(opts.src) + db.listLpush(opts.dst, elem) + c.WriteBulk(elem) + return true + }, + func(c *server.Peer) { + // timeout + c.WriteLen(-1) + }, + ) +} + +// LMOVE +func (m *Miniredis) cmdLmove(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(4)) { + return + } + + opts := struct { + src string + dst string + srcDir string + dstDir string + }{ + src: args[0], + dst: args[1], + srcDir: strings.ToLower(args[2]), + dstDir: strings.ToLower(args[3]), + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.src) { + c.WriteNull() + return + } + if db.t(opts.src) != keyTypeList || (db.exists(opts.dst) && db.t(opts.dst) != keyTypeList) { + c.WriteError(msgWrongType) + return + } + var elem string + switch opts.srcDir { + case "left": + elem = db.listLpop(opts.src) + case "right": + elem = db.listPop(opts.src) + default: + c.WriteError(msgSyntaxError) + return + } + + switch opts.dstDir { + case "left": + db.listLpush(opts.dst, elem) + case "right": + db.listPush(opts.dst, elem) + default: + c.WriteError(msgSyntaxError) + return + } + c.WriteBulk(elem) + }) +} + +// BLMOVE +func (m *Miniredis) cmdBlmove(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(5)) { + return + } + + opts := struct { + src string + dst string + srcDir string + dstDir string + timeout time.Duration + }{ + src: args[0], + dst: args[1], + srcDir: strings.ToLower(args[2]), + dstDir: strings.ToLower(args[3]), + } + if ok := optDuration(c, args[len(args)-1], &opts.timeout); !ok { + return + } + + blocking( + m, + c, + opts.timeout, + func(c *server.Peer, ctx *connCtx) bool { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.src) { + return false + } + if db.t(opts.src) != keyTypeList || (db.exists(opts.dst) && db.t(opts.dst) != keyTypeList) { + c.WriteError(msgWrongType) + return true + } + + var ( + elem string + ttl = db.ttl[opts.src] // in case we empty the array (deletes the entry) + ) + switch opts.srcDir { + case "left": + elem = db.listLpop(opts.src) + case "right": + elem = db.listPop(opts.src) + default: + c.WriteError(msgSyntaxError) + return true + } + + switch opts.dstDir { + case "left": + db.listLpush(opts.dst, elem) + case "right": + db.listPush(opts.dst, elem) + default: + c.WriteError(msgSyntaxError) + return true + } + if ttl > 0 { + db.ttl[opts.dst] = ttl + } + + c.WriteBulk(elem) + return true + }, + func(c *server.Peer) { + // timeout + c.WriteLen(-1) + }, + ) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_object.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_object.go new file mode 100644 index 00000000..e8117c52 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_object.go @@ -0,0 +1,50 @@ +package miniredis + +import ( + "fmt" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsObject handles all object operations. +func commandsObject(m *Miniredis) { + m.srv.Register("OBJECT", m.cmdObject) +} + +// OBJECT +func (m *Miniredis) cmdObject(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + switch sub := strings.ToLower(args[0]); sub { + case "idletime": + m.cmdObjectIdletime(c, args[1:]) + default: + setDirty(c) + c.WriteError(fmt.Sprintf(msgFObjectUsage, sub)) + } +} + +// OBJECT IDLETIME +func (m *Miniredis) cmdObjectIdletime(c *server.Peer, args []string) { + if len(args) != 1 { + setDirty(c) + c.WriteError(errWrongNumber("object|idletime")) + return + } + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + t, ok := db.lru[key] + if !ok { + c.WriteNull() + return + } + + c.WriteInt(int(db.master.effectiveNow().Sub(t).Seconds())) + }) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_pubsub.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_pubsub.go new file mode 100644 index 00000000..571431e8 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_pubsub.go @@ -0,0 +1,254 @@ +// Commands from https://redis.io/commands#pubsub + +package miniredis + +import ( + "fmt" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsPubsub handles all PUB/SUB operations. +func commandsPubsub(m *Miniredis) { + m.srv.Register("SUBSCRIBE", m.cmdSubscribe) + m.srv.Register("UNSUBSCRIBE", m.cmdUnsubscribe) + m.srv.Register("PSUBSCRIBE", m.cmdPsubscribe) + m.srv.Register("PUNSUBSCRIBE", m.cmdPunsubscribe) + m.srv.Register("PUBLISH", m.cmdPublish) + m.srv.Register("PUBSUB", m.cmdPubSub) +} + +// SUBSCRIBE +func (m *Miniredis) cmdSubscribe(c *server.Peer, cmd string, args []string) { + if len(args) < 1 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + if !m.handleAuth(c) { + return + } + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + sub := m.subscribedState(c) + for _, channel := range args { + n := sub.Subscribe(channel) + c.Block(func(w *server.Writer) { + w.WritePushLen(3) + w.WriteBulk("subscribe") + w.WriteBulk(channel) + w.WriteInt(n) + }) + } + }) +} + +// UNSUBSCRIBE +func (m *Miniredis) cmdUnsubscribe(c *server.Peer, cmd string, args []string) { + if !m.handleAuth(c) { + return + } + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + + channels := args + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + sub := m.subscribedState(c) + + if len(channels) == 0 { + channels = sub.Channels() + } + + // there is no de-duplication + for _, channel := range channels { + n := sub.Unsubscribe(channel) + c.Block(func(w *server.Writer) { + w.WritePushLen(3) + w.WriteBulk("unsubscribe") + w.WriteBulk(channel) + w.WriteInt(n) + }) + } + if len(channels) == 0 { + // special case: there is always a reply + c.Block(func(w *server.Writer) { + w.WritePushLen(3) + w.WriteBulk("unsubscribe") + w.WriteNull() + w.WriteInt(0) + }) + } + + if sub.Count() == 0 { + endSubscriber(m, c) + } + }) +} + +// PSUBSCRIBE +func (m *Miniredis) cmdPsubscribe(c *server.Peer, cmd string, args []string) { + if len(args) < 1 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + if !m.handleAuth(c) { + return + } + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + sub := m.subscribedState(c) + for _, pat := range args { + n := sub.Psubscribe(pat) + c.Block(func(w *server.Writer) { + w.WritePushLen(3) + w.WriteBulk("psubscribe") + w.WriteBulk(pat) + w.WriteInt(n) + }) + } + }) +} + +// PUNSUBSCRIBE +func (m *Miniredis) cmdPunsubscribe(c *server.Peer, cmd string, args []string) { + if !m.handleAuth(c) { + return + } + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + + patterns := args + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + sub := m.subscribedState(c) + + if len(patterns) == 0 { + patterns = sub.Patterns() + } + + // there is no de-duplication + for _, pat := range patterns { + n := sub.Punsubscribe(pat) + c.Block(func(w *server.Writer) { + w.WritePushLen(3) + w.WriteBulk("punsubscribe") + w.WriteBulk(pat) + w.WriteInt(n) + }) + } + if len(patterns) == 0 { + // special case: there is always a reply + c.Block(func(w *server.Writer) { + w.WritePushLen(3) + w.WriteBulk("punsubscribe") + w.WriteNull() + w.WriteInt(0) + }) + } + + if sub.Count() == 0 { + endSubscriber(m, c) + } + }) +} + +// PUBLISH +func (m *Miniredis) cmdPublish(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + channel, mesg := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + c.WriteInt(m.publish(channel, mesg)) + }) +} + +// PUBSUB +func (m *Miniredis) cmdPubSub(c *server.Peer, cmd string, args []string) { + if len(args) < 1 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + + if m.checkPubsub(c, cmd) { + return + } + + subcommand := strings.ToUpper(args[0]) + subargs := args[1:] + var argsOk bool + + switch subcommand { + case "CHANNELS": + argsOk = len(subargs) < 2 + case "NUMSUB": + argsOk = true + case "NUMPAT": + argsOk = len(subargs) == 0 + default: + setDirty(c) + c.WriteError(fmt.Sprintf(msgFPubsubUsageSimple, subcommand)) + return + } + + if !argsOk { + setDirty(c) + c.WriteError(fmt.Sprintf(msgFPubsubUsage, subcommand)) + return + } + + if !m.handleAuth(c) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + switch subcommand { + case "CHANNELS": + pat := "" + if len(subargs) == 1 { + pat = subargs[0] + } + + allsubs := m.allSubscribers() + channels := activeChannels(allsubs, pat) + + c.WriteLen(len(channels)) + for _, channel := range channels { + c.WriteBulk(channel) + } + + case "NUMSUB": + subs := m.allSubscribers() + c.WriteLen(len(subargs) * 2) + for _, channel := range subargs { + c.WriteBulk(channel) + c.WriteInt(countSubs(subs, channel)) + } + + case "NUMPAT": + c.WriteInt(countPsubs(m.allSubscribers())) + } + }) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_scripting.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_scripting.go new file mode 100644 index 00000000..5b475c5f --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_scripting.go @@ -0,0 +1,345 @@ +package miniredis + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "strconv" + "strings" + "sync" + + lua "github.com/yuin/gopher-lua" + "github.com/yuin/gopher-lua/parse" + + luajson "github.com/alicebob/miniredis/v2/gopher-json" + "github.com/alicebob/miniredis/v2/server" +) + +func commandsScripting(m *Miniredis) { + m.srv.Register("EVAL", m.cmdEval) + m.srv.Register("EVAL_RO", m.cmdEvalro, server.ReadOnlyOption()) + m.srv.Register("EVALSHA", m.cmdEvalsha) + m.srv.Register("EVALSHA_RO", m.cmdEvalshaRo, server.ReadOnlyOption()) + m.srv.Register("SCRIPT", m.cmdScript) +} + +var ( + parsedScripts = sync.Map{} +) + +// Execute lua. Needs to run m.Lock()ed, from within withTx(). +// Returns true if the lua was OK (and hence should be cached). +func (m *Miniredis) runLuaScript(c *server.Peer, sha, script string, readOnly bool, args []string) bool { + l := lua.NewState(lua.Options{SkipOpenLibs: true}) + defer l.Close() + + // Taken from the go-lua manual + for _, pair := range []struct { + n string + f lua.LGFunction + }{ + {lua.LoadLibName, lua.OpenPackage}, + {lua.BaseLibName, lua.OpenBase}, + {lua.CoroutineLibName, lua.OpenCoroutine}, + {lua.TabLibName, lua.OpenTable}, + {lua.StringLibName, lua.OpenString}, + {lua.MathLibName, lua.OpenMath}, + {lua.DebugLibName, lua.OpenDebug}, + } { + if err := l.CallByParam(lua.P{ + Fn: l.NewFunction(pair.f), + NRet: 0, + Protect: true, + }, lua.LString(pair.n)); err != nil { + panic(err) + } + } + + luajson.Preload(l) + requireGlobal(l, "cjson", "json") + + // set global variable KEYS + keysTable := l.NewTable() + keysS, args := args[0], args[1:] + keysLen, err := strconv.Atoi(keysS) + if err != nil { + c.WriteError(msgInvalidInt) + return false + } + if keysLen < 0 { + c.WriteError(msgNegativeKeysNumber) + return false + } + if keysLen > len(args) { + c.WriteError(msgInvalidKeysNumber) + return false + } + keys, args := args[:keysLen], args[keysLen:] + for i, k := range keys { + l.RawSet(keysTable, lua.LNumber(i+1), lua.LString(k)) + } + l.SetGlobal("KEYS", keysTable) + + argvTable := l.NewTable() + for i, a := range args { + l.RawSet(argvTable, lua.LNumber(i+1), lua.LString(a)) + } + l.SetGlobal("ARGV", argvTable) + + redisFuncs, redisConstants := mkLua(m.srv, c, sha, readOnly) + redisMod := l.CreateTable(0, len(redisFuncs)+len(redisConstants)) + for fname, fn := range redisFuncs { + redisMod.RawSetString(fname, l.NewFunction(fn)) + } + for k, v := range redisConstants { + redisMod.RawSetString(k, v) + } + for _, name := range []string{"redis", "server"} { + l.SetGlobal(name, redisMod) + } + + l.RegisterModule("os", mkLuaOS()) + + _ = doScript(l, protectGlobals) + + // lua can call redis.setresp(...), but it's tmp state. + oldresp := c.Resp3 + if err := doScript(l, script); err != nil { + c.WriteError(err.Error()) + return false + } + + luaToRedis(l, c, l.Get(1)) + c.Resp3 = oldresp + c.SwitchResp3 = nil + return true +} + +// doScript pre-compiles the given script into a Lua prototype, +// then executes the pre-compiled function against the given lua state. +// +// This is thread-safe. +func doScript(l *lua.LState, script string) error { + proto, err := compile(script) + if err != nil { + return fmt.Errorf(errLuaParseError(err)) + } + + lfunc := l.NewFunctionFromProto(proto) + l.Push(lfunc) + if err := l.PCall(0, lua.MultRet, nil); err != nil { + // ensure we wrap with the correct format. + return fmt.Errorf(errLuaParseError(err)) + } + + return nil +} + +func compile(script string) (*lua.FunctionProto, error) { + if val, ok := parsedScripts.Load(script); ok { + return val.(*lua.FunctionProto), nil + } + chunk, err := parse.Parse(strings.NewReader(script), "") + if err != nil { + return nil, err + } + proto, err := lua.Compile(chunk, "") + if err != nil { + return nil, err + } + parsedScripts.Store(script, proto) + return proto, nil +} + +// Shared implementation for EVAL and EVALRO +func (m *Miniredis) cmdEvalShared(c *server.Peer, cmd string, readOnly bool, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + + script, args := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + sha := sha1Hex(script) + ok := m.runLuaScript(c, sha, script, readOnly, args) + if ok { + m.scripts[sha] = script + } + }) +} + +// Wrapper function for EVAL command +func (m *Miniredis) cmdEval(c *server.Peer, cmd string, args []string) { + m.cmdEvalShared(c, cmd, false, args) +} + +// Shared implementation for EVALSHA and EVALSHA_RO +func (m *Miniredis) cmdEvalshaShared(c *server.Peer, cmd string, readOnly bool, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + + sha, args := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + script, ok := m.scripts[sha] + if !ok { + c.WriteError(msgNoScriptFound) + return + } + + m.runLuaScript(c, sha, script, readOnly, args) + }) +} + +// Wrapper function for EVALSHA command +func (m *Miniredis) cmdEvalsha(c *server.Peer, cmd string, args []string) { + m.cmdEvalshaShared(c, cmd, false, args) +} + +// Wrapper function for EVALRO command +func (m *Miniredis) cmdEvalro(c *server.Peer, cmd string, args []string) { + m.cmdEvalShared(c, cmd, true, args) +} + +// Wrapper function for EVALSHA_RO command +func (m *Miniredis) cmdEvalshaRo(c *server.Peer, cmd string, args []string) { + m.cmdEvalshaShared(c, cmd, true, args) +} + +func (m *Miniredis) cmdScript(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + + var opts struct { + subcmd string + script string + } + + opts.subcmd, args = args[0], args[1:] + + switch strings.ToLower(opts.subcmd) { + case "load": + if len(args) != 1 { + setDirty(c) + c.WriteError(fmt.Sprintf(msgFScriptUsage, "LOAD")) + return + } + opts.script = args[0] + case "exists": + if len(args) == 0 { + setDirty(c) + c.WriteError(errWrongNumber("script|exists")) + return + } + case "flush": + if len(args) == 1 { + switch strings.ToUpper(args[0]) { + case "SYNC", "ASYNC": + args = args[1:] + default: + } + } + if len(args) != 0 { + setDirty(c) + c.WriteError(msgScriptFlush) + return + } + default: + setDirty(c) + c.WriteError(fmt.Sprintf(msgFScriptUsageSimple, strings.ToUpper(opts.subcmd))) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + switch strings.ToLower(opts.subcmd) { + case "load": + if _, err := parse.Parse(strings.NewReader(opts.script), "user_script"); err != nil { + c.WriteError(errLuaParseError(err)) + return + } + sha := sha1Hex(opts.script) + m.scripts[sha] = opts.script + c.WriteBulk(sha) + case "exists": + c.WriteLen(len(args)) + for _, arg := range args { + if _, ok := m.scripts[arg]; ok { + c.WriteInt(1) + } else { + c.WriteInt(0) + } + } + case "flush": + m.scripts = map[string]string{} + c.WriteOK() + } + }) +} + +func sha1Hex(s string) string { + h := sha1.New() + io.WriteString(h, s) + return hex.EncodeToString(h.Sum(nil)) +} + +// requireGlobal imports module modName into the global namespace with the +// identifier id. panics if an error results from the function execution +func requireGlobal(l *lua.LState, id, modName string) { + if err := l.CallByParam(lua.P{ + Fn: l.GetGlobal("require"), + NRet: 1, + Protect: true, + }, lua.LString(modName)); err != nil { + panic(err) + } + mod := l.Get(-1) + l.Pop(1) + + l.SetGlobal(id, mod) +} + +// the following script protects globals +// it is based on: http://metalua.luaforge.net/src/lib/strict.lua.html +var protectGlobals = ` +local dbg=debug +local mt = {} +setmetatable(_G, mt) +mt.__newindex = function (t, n, v) + if dbg.getinfo(2) then + local w = dbg.getinfo(2, "S").what + if w ~= "C" then + error("Script attempted to create global variable '"..tostring(n).."'", 2) + end + end + rawset(t, n, v) +end +mt.__index = function (t, n) + if dbg.getinfo(2) and dbg.getinfo(2, "S").what ~= "C" then + error("Script attempted to access nonexistent global variable '"..tostring(n).."'", 2) + end + return rawget(t, n) +end +debug = nil + +` diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_server.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_server.go new file mode 100644 index 00000000..79e19fbc --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_server.go @@ -0,0 +1,153 @@ +// Commands from https://redis.io/commands#server + +package miniredis + +import ( + "fmt" + "strconv" + "strings" + + "github.com/alicebob/miniredis/v2/server" + "github.com/alicebob/miniredis/v2/size" +) + +func commandsServer(m *Miniredis) { + m.srv.Register("COMMAND", m.cmdCommand) + m.srv.Register("DBSIZE", m.cmdDbsize, server.ReadOnlyOption()) + m.srv.Register("FLUSHALL", m.cmdFlushall) + m.srv.Register("FLUSHDB", m.cmdFlushdb) + m.srv.Register("INFO", m.cmdInfo) + m.srv.Register("TIME", m.cmdTime) + m.srv.Register("MEMORY", m.cmdMemory) +} + +// MEMORY +func (m *Miniredis) cmdMemory(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + cmd, args := strings.ToLower(args[0]), args[1:] + switch cmd { + case "usage": + if len(args) < 1 { + setDirty(c) + c.WriteError(errWrongNumber("memory|usage")) + return + } + if len(args) > 1 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + var ( + value interface{} + ok bool + ) + switch db.keys[args[0]] { + case keyTypeString: + value, ok = db.stringKeys[args[0]] + case keyTypeSet: + value, ok = db.setKeys[args[0]] + case keyTypeHash: + value, ok = db.hashKeys[args[0]] + case keyTypeList: + value, ok = db.listKeys[args[0]] + case keyTypeHll: + value, ok = db.hllKeys[args[0]] + case keyTypeSortedSet: + value, ok = db.sortedsetKeys[args[0]] + case keyTypeStream: + value, ok = db.streamKeys[args[0]] + } + if !ok { + c.WriteNull() + return + } + c.WriteInt(size.Of(value)) + default: + c.WriteError(fmt.Sprintf(msgMemorySubcommand, strings.ToUpper(cmd))) + } + }) +} + +// DBSIZE +func (m *Miniredis) cmdDbsize(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(0)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + c.WriteInt(len(db.keys)) + }) +} + +// FLUSHALL +func (m *Miniredis) cmdFlushall(c *server.Peer, cmd string, args []string) { + if len(args) > 0 && strings.ToLower(args[0]) == "async" { + args = args[1:] + } + if len(args) > 0 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + if !m.handleAuth(c) { + return + } + if m.checkPubsub(c, cmd) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + m.flushAll() + c.WriteOK() + }) +} + +// FLUSHDB +func (m *Miniredis) cmdFlushdb(c *server.Peer, cmd string, args []string) { + if len(args) > 0 && strings.ToLower(args[0]) == "async" { + args = args[1:] + } + if len(args) > 0 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + if !m.handleAuth(c) { + return + } + if m.checkPubsub(c, cmd) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + m.db(ctx.selectedDB).flush() + c.WriteOK() + }) +} + +// TIME +func (m *Miniredis) cmdTime(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(0)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + now := m.effectiveNow() + nanos := now.UnixNano() + seconds := nanos / 1_000_000_000 + microseconds := (nanos / 1_000) % 1_000_000 + + c.WriteLen(2) + c.WriteBulk(strconv.FormatInt(seconds, 10)) + c.WriteBulk(strconv.FormatInt(microseconds, 10)) + }) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_set.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_set.go new file mode 100644 index 00000000..ec35d563 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_set.go @@ -0,0 +1,701 @@ +// Commands from https://redis.io/commands#set + +package miniredis + +import ( + "fmt" + "strconv" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsSet handles all set value operations. +func commandsSet(m *Miniredis) { + m.srv.Register("SADD", m.cmdSadd) + m.srv.Register("SCARD", m.cmdScard, server.ReadOnlyOption()) + m.srv.Register("SDIFF", m.cmdSdiff, server.ReadOnlyOption()) + m.srv.Register("SDIFFSTORE", m.cmdSdiffstore) + m.srv.Register("SINTERCARD", m.cmdSintercard, server.ReadOnlyOption()) + m.srv.Register("SINTER", m.cmdSinter, server.ReadOnlyOption()) + m.srv.Register("SINTERSTORE", m.cmdSinterstore) + m.srv.Register("SISMEMBER", m.cmdSismember, server.ReadOnlyOption()) + m.srv.Register("SMEMBERS", m.cmdSmembers, server.ReadOnlyOption()) + m.srv.Register("SMISMEMBER", m.cmdSmismember, server.ReadOnlyOption()) + m.srv.Register("SMOVE", m.cmdSmove) + m.srv.Register("SPOP", m.cmdSpop) + m.srv.Register("SRANDMEMBER", m.cmdSrandmember, server.ReadOnlyOption()) + m.srv.Register("SREM", m.cmdSrem) + m.srv.Register("SUNION", m.cmdSunion, server.ReadOnlyOption()) + m.srv.Register("SUNIONSTORE", m.cmdSunionstore) + m.srv.Register("SSCAN", m.cmdSscan, server.ReadOnlyOption()) +} + +// SADD +func (m *Miniredis) cmdSadd(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, elems := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if db.exists(key) && db.t(key) != keyTypeSet { + c.WriteError(ErrWrongType.Error()) + return + } + + added := db.setAdd(key, elems...) + c.WriteInt(added) + }) +} + +// SCARD +func (m *Miniredis) cmdScard(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteInt(0) + return + } + + if db.t(key) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.setMembers(key) + c.WriteInt(len(members)) + }) +} + +// SDIFF +func (m *Miniredis) cmdSdiff(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + keys := args + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + set, err := db.setDiff(keys) + if err != nil { + c.WriteError(err.Error()) + return + } + + c.WriteSetLen(len(set)) + for k := range set { + c.WriteBulk(k) + } + }) +} + +// SDIFFSTORE +func (m *Miniredis) cmdSdiffstore(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + dest, keys := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + set, err := db.setDiff(keys) + if err != nil { + c.WriteError(err.Error()) + return + } + + db.del(dest, true) + db.setSet(dest, set) + c.WriteInt(len(set)) + }) +} + +// SINTER +func (m *Miniredis) cmdSinter(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + keys := args + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + set, err := db.setInter(keys) + if err != nil { + c.WriteError(err.Error()) + return + } + + c.WriteLen(len(set)) + for k := range set { + c.WriteBulk(k) + } + }) +} + +// SINTERSTORE +func (m *Miniredis) cmdSinterstore(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + dest, keys := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + set, err := db.setInter(keys) + if err != nil { + c.WriteError(err.Error()) + return + } + + db.del(dest, true) + db.setSet(dest, set) + c.WriteInt(len(set)) + }) +} + +// SINTERCARD +func (m *Miniredis) cmdSintercard(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + opts := struct { + keys []string + limit int + }{} + + numKeys, err := strconv.Atoi(args[0]) + if err != nil { + setDirty(c) + c.WriteError("ERR numkeys should be greater than 0") + return + } + if numKeys < 1 { + setDirty(c) + c.WriteError("ERR numkeys should be greater than 0") + return + } + + args = args[1:] + if len(args) < numKeys { + setDirty(c) + c.WriteError("ERR Number of keys can't be greater than number of args") + return + } + opts.keys = args[:numKeys] + + args = args[numKeys:] + if len(args) == 2 && strings.ToLower(args[0]) == "limit" { + l, err := strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if l < 0 { + setDirty(c) + c.WriteError(msgLimitIsNegative) + return + } + opts.limit = l + } else if len(args) > 0 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + count, err := db.setIntercard(opts.keys, opts.limit) + if err != nil { + c.WriteError(err.Error()) + return + } + c.WriteInt(count) + }) +} + +// SISMEMBER +func (m *Miniredis) cmdSismember(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + key, value := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteInt(0) + return + } + + if db.t(key) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + if db.setIsMember(key, value) { + c.WriteInt(1) + return + } + c.WriteInt(0) + }) +} + +// SMEMBERS +func (m *Miniredis) cmdSmembers(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteSetLen(0) + return + } + + if db.t(key) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.setMembers(key) + + c.WriteSetLen(len(members)) + for _, elem := range members { + c.WriteBulk(elem) + } + }) +} + +// SMISMEMBER +func (m *Miniredis) cmdSmismember(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, values := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteLen(len(values)) + for range values { + c.WriteInt(0) + } + return + } + + if db.t(key) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + c.WriteLen(len(values)) + for _, value := range values { + if db.setIsMember(key, value) { + c.WriteInt(1) + } else { + c.WriteInt(0) + } + } + return + }) +} + +// SMOVE +func (m *Miniredis) cmdSmove(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + src, dst, member := args[0], args[1], args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(src) { + c.WriteInt(0) + return + } + + if db.t(src) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + if db.exists(dst) && db.t(dst) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + if !db.setIsMember(src, member) { + c.WriteInt(0) + return + } + db.setRem(src, member) + db.setAdd(dst, member) + c.WriteInt(1) + }) +} + +// SPOP +func (m *Miniredis) cmdSpop(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + opts := struct { + key string + withCount bool + count int + }{ + count: 1, + } + opts.key, args = args[0], args[1:] + + if len(args) > 0 { + v, err := strconv.Atoi(args[0]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if v < 0 { + setDirty(c) + c.WriteError(msgOutOfRange) + return + } + opts.count = v + opts.withCount = true + args = args[1:] + } + if len(args) > 0 { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + if !opts.withCount { + c.WriteNull() + return + } + c.WriteLen(0) + return + } + + if db.t(opts.key) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + var deleted []string + members := db.setMembers(opts.key) + for i := 0; i < opts.count; i++ { + if len(members) == 0 { + break + } + i := m.randIntn(len(members)) + member := members[i] + members = delElem(members, i) + db.setRem(opts.key, member) + deleted = append(deleted, member) + } + // without `count` return a single value + if !opts.withCount { + if len(deleted) == 0 { + c.WriteNull() + return + } + c.WriteBulk(deleted[0]) + return + } + // with `count` return a list + c.WriteLen(len(deleted)) + for _, v := range deleted { + c.WriteBulk(v) + } + }) +} + +// SRANDMEMBER +func (m *Miniredis) cmdSrandmember(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + if len(args) > 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + key := args[0] + count := 0 + withCount := false + if len(args) == 2 { + var err error + count, err = strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + withCount = true + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + if withCount { + c.WriteLen(0) + return + } + c.WriteNull() + return + } + + if db.t(key) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.setMembers(key) + if count < 0 { + // Non-unique elements is allowed with negative count. + c.WriteLen(-count) + for count != 0 { + member := members[m.randIntn(len(members))] + c.WriteBulk(member) + count++ + } + return + } + + // Must be unique elements. + m.shuffle(members) + if count > len(members) { + count = len(members) + } + if !withCount { + c.WriteBulk(members[0]) + return + } + c.WriteLen(count) + for i := range make([]struct{}, count) { + c.WriteBulk(members[i]) + } + }) +} + +// SREM +func (m *Miniredis) cmdSrem(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, fields := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteInt(0) + return + } + + if db.t(key) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + + c.WriteInt(db.setRem(key, fields...)) + }) +} + +// SUNION +func (m *Miniredis) cmdSunion(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + keys := args + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + set, err := db.setUnion(keys) + if err != nil { + c.WriteError(err.Error()) + return + } + + c.WriteLen(len(set)) + for k := range set { + c.WriteBulk(k) + } + }) +} + +// SUNIONSTORE +func (m *Miniredis) cmdSunionstore(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + dest, keys := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + set, err := db.setUnion(keys) + if err != nil { + c.WriteError(err.Error()) + return + } + + db.del(dest, true) + db.setSet(dest, set) + c.WriteInt(len(set)) + }) +} + +// SSCAN +func (m *Miniredis) cmdSscan(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + var opts struct { + key string + value int + cursor int + count int + withMatch bool + match string + } + + opts.key = args[0] + if ok := optIntErr(c, args[1], &opts.cursor, msgInvalidCursor); !ok { + return + } + args = args[2:] + + // MATCH and COUNT options + for len(args) > 0 { + if strings.ToLower(args[0]) == "count" { + if len(args) < 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + count, err := strconv.Atoi(args[1]) + if err != nil || count < 0 { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if count == 0 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + opts.count = count + args = args[2:] + continue + } + if strings.ToLower(args[0]) == "match" { + if len(args) < 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + opts.withMatch = true + opts.match = args[1] + args = args[2:] + continue + } + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + // return _all_ (matched) keys every time + if db.exists(opts.key) && db.t(opts.key) != "set" { + c.WriteError(ErrWrongType.Error()) + return + } + members := db.setMembers(opts.key) + if opts.withMatch { + members, _ = matchKeys(members, opts.match) + } + low := opts.cursor + high := low + opts.count + // validate high is correct + if high > len(members) || high == 0 { + high = len(members) + } + if opts.cursor > high { + // invalid cursor + c.WriteLen(2) + c.WriteBulk("0") // no next cursor + c.WriteLen(0) // no elements + return + } + cursorValue := low + opts.count + if cursorValue > len(members) { + cursorValue = 0 // no next cursor + } + members = members[low:high] + c.WriteLen(2) + c.WriteBulk(fmt.Sprintf("%d", cursorValue)) + c.WriteLen(len(members)) + for _, k := range members { + c.WriteBulk(k) + } + + }) +} + +func delElem(ls []string, i int) []string { + // this swap+truncate is faster but changes behaviour: + // ls[i] = ls[len(ls)-1] + // ls = ls[:len(ls)-1] + // so we do the dumb thing: + ls = append(ls[:i], ls[i+1:]...) + return ls +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_sorted_set.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_sorted_set.go new file mode 100644 index 00000000..1491a74f --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_sorted_set.go @@ -0,0 +1,1857 @@ +// Commands from https://redis.io/commands#sorted_set + +package miniredis + +import ( + "errors" + "fmt" + "math" + "sort" + "strconv" + "strings" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsSortedSet handles all sorted set operations. +func commandsSortedSet(m *Miniredis) { + m.srv.Register("ZADD", m.cmdZadd) + m.srv.Register("ZCARD", m.cmdZcard, server.ReadOnlyOption()) + m.srv.Register("ZCOUNT", m.cmdZcount, server.ReadOnlyOption()) + m.srv.Register("ZINCRBY", m.cmdZincrby) + m.srv.Register("ZINTER", m.makeCmdZinter(false), server.ReadOnlyOption()) + m.srv.Register("ZINTERSTORE", m.makeCmdZinter(true)) + m.srv.Register("ZLEXCOUNT", m.cmdZlexcount, server.ReadOnlyOption()) + m.srv.Register("ZRANGE", m.cmdZrange, server.ReadOnlyOption()) + m.srv.Register("ZRANGEBYLEX", m.makeCmdZrangebylex(false), server.ReadOnlyOption()) + m.srv.Register("ZRANGEBYSCORE", m.makeCmdZrangebyscore(false), server.ReadOnlyOption()) + m.srv.Register("ZRANK", m.makeCmdZrank(false), server.ReadOnlyOption()) + m.srv.Register("ZREM", m.cmdZrem) + m.srv.Register("ZREMRANGEBYLEX", m.cmdZremrangebylex) + m.srv.Register("ZREMRANGEBYRANK", m.cmdZremrangebyrank) + m.srv.Register("ZREMRANGEBYSCORE", m.cmdZremrangebyscore) + m.srv.Register("ZREVRANGE", m.cmdZrevrange, server.ReadOnlyOption()) + m.srv.Register("ZREVRANGEBYLEX", m.makeCmdZrangebylex(true), server.ReadOnlyOption()) + m.srv.Register("ZREVRANGEBYSCORE", m.makeCmdZrangebyscore(true), server.ReadOnlyOption()) + m.srv.Register("ZREVRANK", m.makeCmdZrank(true), server.ReadOnlyOption()) + m.srv.Register("ZSCORE", m.cmdZscore, server.ReadOnlyOption()) + m.srv.Register("ZMSCORE", m.cmdZMscore, server.ReadOnlyOption()) + m.srv.Register("ZUNION", m.cmdZunion, server.ReadOnlyOption()) + m.srv.Register("ZUNIONSTORE", m.cmdZunionstore) + m.srv.Register("ZSCAN", m.cmdZscan, server.ReadOnlyOption()) + m.srv.Register("ZPOPMAX", m.cmdZpopmax(true)) + m.srv.Register("ZPOPMIN", m.cmdZpopmax(false)) + m.srv.Register("ZRANDMEMBER", m.cmdZrandmember, server.ReadOnlyOption()) +} + +// ZADD +func (m *Miniredis) cmdZadd(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + var opts struct { + key string + nx bool + xx bool + gt bool + lt bool + ch bool + incr bool + } + elems := map[string]float64{} + + opts.key = args[0] + args = args[1:] +outer: + for len(args) > 0 { + switch strings.ToUpper(args[0]) { + case "NX": + opts.nx = true + args = args[1:] + continue + case "XX": + opts.xx = true + args = args[1:] + continue + case "GT": + opts.gt = true + args = args[1:] + continue + case "LT": + opts.lt = true + args = args[1:] + continue + case "CH": + opts.ch = true + args = args[1:] + continue + case "INCR": + opts.incr = true + args = args[1:] + continue + default: + break outer + } + } + + if len(args) == 0 || len(args)%2 != 0 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + for len(args) > 0 { + score, err := strconv.ParseFloat(args[0], 64) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidFloat) + return + } + elems[args[1]] = score + args = args[2:] + } + + if opts.xx && opts.nx { + setDirty(c) + c.WriteError(msgXXandNX) + return + } + + if opts.gt && opts.lt || + opts.gt && opts.nx || + opts.lt && opts.nx { + setDirty(c) + c.WriteError(msgGTLTandNX) + return + } + + if opts.incr && len(elems) > 1 { + setDirty(c) + c.WriteError(msgSingleElementPair) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if db.exists(opts.key) && db.t(opts.key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + if opts.incr { + for member, delta := range elems { + if opts.nx && db.ssetExists(opts.key, member) { + c.WriteNull() + return + } + if opts.xx && !db.ssetExists(opts.key, member) { + c.WriteNull() + return + } + newScore := db.ssetIncrby(opts.key, member, delta) + c.WriteFloat(newScore) + } + return + } + + res := 0 + for member, score := range elems { + exists := db.ssetExists(opts.key, member) + if opts.nx && exists { + continue + } + if opts.xx && !exists { + continue + } + old := db.ssetScore(opts.key, member) + if opts.gt && exists && score <= old { + continue + } + if opts.lt && exists && score >= old { + continue + } + if db.ssetAdd(opts.key, score, member) { + res++ + } else { + if opts.ch && old != score { + // if 'CH' is specified, only count changed keys + res++ + } + } + } + c.WriteInt(res) + }) +} + +// ZCARD +func (m *Miniredis) cmdZcard(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteInt(0) + return + } + + if db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + c.WriteInt(db.ssetCard(key)) + }) +} + +// ZCOUNT +func (m *Miniredis) cmdZcount(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var ( + opts struct { + key string + min float64 + minIncl bool + max float64 + maxIncl bool + } + err error + ) + + opts.key = args[0] + opts.min, opts.minIncl, err = parseFloatRange(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidMinMax) + return + } + opts.max, opts.maxIncl, err = parseFloatRange(args[2]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidMinMax) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + c.WriteInt(0) + return + } + + if db.t(opts.key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetElements(opts.key) + members = withSSRange(members, opts.min, opts.minIncl, opts.max, opts.maxIncl) + c.WriteInt(len(members)) + }) +} + +// ZINCRBY +func (m *Miniredis) cmdZincrby(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + delta float64 + member string + } + + opts.key = args[0] + d, err := strconv.ParseFloat(args[1], 64) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidFloat) + return + } + opts.delta = d + opts.member = args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if db.exists(opts.key) && db.t(opts.key) != keyTypeSortedSet { + c.WriteError(msgWrongType) + return + } + newScore := db.ssetIncrby(opts.key, opts.member, opts.delta) + c.WriteFloat(newScore) + }) +} + +// ZINTERSTORE and ZINTER +func (m *Miniredis) makeCmdZinter(store bool) func(c *server.Peer, cmd string, args []string) { + return func(c *server.Peer, cmd string, args []string) { + minArgs := 2 + if store { + minArgs++ + } + if !m.isValidCMD(c, cmd, args, atLeast(minArgs)) { + return + } + + var opts = struct { + Store bool // if true this is ZINTERSTORE + Destination string // only relevant if $store is true + Keys []string + Aggregate string + WithWeights bool + Weights []float64 + WithScores bool // only for ZINTER + }{ + Store: store, + Aggregate: "sum", + } + + if store { + opts.Destination = args[0] + args = args[1:] + } + numKeys, err := strconv.Atoi(args[0]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + args = args[1:] + if len(args) < numKeys { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + if numKeys <= 0 { + setDirty(c) + c.WriteError("ERR at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE") + return + } + opts.Keys = args[:numKeys] + args = args[numKeys:] + + for len(args) > 0 { + switch strings.ToLower(args[0]) { + case "weights": + if len(args) < numKeys+1 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + for i := 0; i < numKeys; i++ { + f, err := strconv.ParseFloat(args[i+1], 64) + if err != nil { + setDirty(c) + c.WriteError("ERR weight value is not a float") + return + } + opts.Weights = append(opts.Weights, f) + } + opts.WithWeights = true + args = args[numKeys+1:] + case "aggregate": + if len(args) < 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + aggregate := strings.ToLower(args[1]) + switch aggregate { + case "sum", "min", "max": + opts.Aggregate = aggregate + default: + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + args = args[2:] + case "withscores": + if store { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + opts.WithScores = true + args = args[1:] + default: + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + // We collect everything and remove all keys which turned out not to be + // present in every set. + sset := map[string]float64{} + counts := map[string]int{} + for i, key := range opts.Keys { + if !db.exists(key) { + continue + } + + var set map[string]float64 + switch db.t(key) { + case keyTypeSet: + set = map[string]float64{} + for elem := range db.setKeys[key] { + set[elem] = 1.0 + } + case keyTypeSortedSet: + set = db.sortedSet(key) + default: + c.WriteError(msgWrongType) + return + } + for member, score := range set { + if opts.WithWeights { + score *= opts.Weights[i] + } + counts[member]++ + old, ok := sset[member] + if !ok { + sset[member] = score + continue + } + switch opts.Aggregate { + default: + panic("Invalid aggregate") + case "sum": + sset[member] += score + case "min": + if score < old { + sset[member] = score + } + case "max": + if score > old { + sset[member] = score + } + } + } + } + for key, count := range counts { + if count != numKeys { + delete(sset, key) + } + } + + if opts.Store { + // ZINTERSTORE mode + db.del(opts.Destination, true) + db.ssetSet(opts.Destination, sset) + c.WriteInt(len(sset)) + return + } + // ZINTER mode + size := len(sset) + if opts.WithScores { + size *= 2 + } + c.WriteLen(size) + for _, l := range sortedKeys(sset) { + c.WriteBulk(l) + if opts.WithScores { + c.WriteFloat(sset[l]) + } + } + }) + } +} + +// ZLEXCOUNT +func (m *Miniredis) cmdZlexcount(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts = struct { + Key string + Min string + Max string + }{ + Key: args[0], + Min: args[1], + Max: args[2], + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + min, minIncl, minErr := parseLexrange(opts.Min) + max, maxIncl, maxErr := parseLexrange(opts.Max) + if minErr != nil || maxErr != nil { + c.WriteError(msgInvalidRangeItem) + return + } + + db := m.db(ctx.selectedDB) + + if !db.exists(opts.Key) { + c.WriteInt(0) + return + } + + if db.t(opts.Key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetMembers(opts.Key) + // Just key sort. If scores are not the same we don't care. + sort.Strings(members) + members = withLexRange(members, min, minIncl, max, maxIncl) + + c.WriteInt(len(members)) + }) +} + +// ZRANGE +func (m *Miniredis) cmdZrange(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + var opts struct { + Key string + Min string + Max string + WithScores bool + ByScore bool + ByLex bool + Reverse bool + WithLimit bool + Offset string + Count string + } + + opts.Key, opts.Min, opts.Max = args[0], args[1], args[2] + args = args[3:] + + for len(args) > 0 { + switch strings.ToLower(args[0]) { + case "byscore": + opts.ByScore = true + args = args[1:] + case "bylex": + opts.ByLex = true + args = args[1:] + case "rev": + opts.Reverse = true + args = args[1:] + case "limit": + opts.WithLimit = true + args = args[1:] + if len(args) < 2 { + c.WriteError(msgSyntaxError) + return + } + opts.Offset = args[0] + opts.Count = args[1] + args = args[2:] + case "withscores": + opts.WithScores = true + args = args[1:] + default: + c.WriteError(msgSyntaxError) + return + } + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + switch { + case opts.ByScore && opts.ByLex: + c.WriteError(msgSyntaxError) + case opts.ByScore: + runRangeByScore(m, c, ctx, optsRangeByScore{ + Key: opts.Key, + Min: opts.Min, + Max: opts.Max, + Reverse: opts.Reverse, + WithLimit: opts.WithLimit, + Offset: opts.Offset, + Count: opts.Count, + WithScores: opts.WithScores, + }) + case opts.ByLex: + runRangeByLex(m, c, ctx, optsRangeByLex{ + Key: opts.Key, + Min: opts.Min, + Max: opts.Max, + Reverse: opts.Reverse, + WithLimit: opts.WithLimit, + Offset: opts.Offset, + Count: opts.Count, + WithScores: opts.WithScores, + }) + default: + if opts.WithLimit { + c.WriteError(msgLimitCombination) + return + } + runRange(m, c, ctx, optsRange{ + Key: opts.Key, + Min: opts.Min, + Max: opts.Max, + Reverse: opts.Reverse, + WithScores: opts.WithScores, + }) + } + }) +} + +// ZREVRANGE +func (m *Miniredis) cmdZrevrange(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + var opts = optsRange{ + Reverse: true, + Key: args[0], + Min: args[1], + Max: args[2], + } + args = args[3:] + + for len(args) > 0 { + switch strings.ToLower(args[0]) { + case "withscores": + opts.WithScores = true + args = args[1:] + default: + c.WriteError(msgSyntaxError) + return + } + } + + withTx(m, c, func(c *server.Peer, cctx *connCtx) { + runRange(m, c, cctx, opts) + }) +} + +// ZRANGEBYLEX and ZREVRANGEBYLEX +func (m *Miniredis) makeCmdZrangebylex(reverse bool) server.Cmd { + return func(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + opts := optsRangeByLex{ + Reverse: reverse, + Key: args[0], + Min: args[1], + Max: args[2], + } + args = args[3:] + + for len(args) > 0 { + switch strings.ToLower(args[0]) { + case "limit": + opts.WithLimit = true + args = args[1:] + if len(args) < 2 { + c.WriteError(msgSyntaxError) + return + } + opts.Offset = args[0] + opts.Count = args[1] + args = args[2:] + continue + default: + // Syntax error + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + } + + withTx(m, c, func(c *server.Peer, cctx *connCtx) { + runRangeByLex(m, c, cctx, opts) + }) + } +} + +// ZRANGEBYSCORE and ZREVRANGEBYSCORE +func (m *Miniredis) makeCmdZrangebyscore(reverse bool) server.Cmd { + return func(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + var opts = optsRangeByScore{ + Reverse: reverse, + Key: args[0], + Min: args[1], + Max: args[2], + } + args = args[3:] + + for len(args) > 0 { + if strings.ToLower(args[0]) == "limit" { + opts.WithLimit = true + args = args[1:] + if len(args) < 2 { + c.WriteError(msgSyntaxError) + return + } + opts.Offset = args[0] + opts.Count = args[1] + args = args[2:] + continue + } + if strings.ToLower(args[0]) == "withscores" { + opts.WithScores = true + args = args[1:] + continue + } + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, cctx *connCtx) { + runRangeByScore(m, c, cctx, opts) + }) + } +} + +// ZRANK and ZREVRANK +func (m *Miniredis) makeCmdZrank(reverse bool) server.Cmd { + return func(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, member := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + withScore := false + if len(args) > 0 && strings.ToUpper(args[len(args)-1]) == "WITHSCORE" { + withScore = true + args = args[:len(args)-1] + } + + if len(args) > 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + if !db.exists(key) { + if withScore { + c.WriteLen(-1) + } else { + c.WriteNull() + } + return + } + + if db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + direction := asc + if reverse { + direction = desc + } + rank, ok := db.ssetRank(key, member, direction) + if !ok { + if withScore { + c.WriteLen(-1) + } else { + c.WriteNull() + } + return + } + + if withScore { + c.WriteLen(2) + c.WriteInt(rank) + c.WriteFloat(db.ssetScore(key, member)) + } else { + c.WriteInt(rank) + } + }) + } +} + +// ZREM +func (m *Miniredis) cmdZrem(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, members := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteInt(0) + return + } + + if db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + deleted := 0 + for _, member := range members { + if db.ssetRem(key, member) { + deleted++ + } + } + c.WriteInt(deleted) + }) +} + +// ZREMRANGEBYLEX +func (m *Miniredis) cmdZremrangebylex(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts = struct { + Key string + Min string + Max string + }{ + Key: args[0], + Min: args[1], + Max: args[2], + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + min, minIncl, minErr := parseLexrange(opts.Min) + max, maxIncl, maxErr := parseLexrange(opts.Max) + if minErr != nil || maxErr != nil { + c.WriteError(msgInvalidRangeItem) + return + } + + db := m.db(ctx.selectedDB) + + if !db.exists(opts.Key) { + c.WriteInt(0) + return + } + + if db.t(opts.Key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetMembers(opts.Key) + // Just key sort. If scores are not the same we don't care. + sort.Strings(members) + members = withLexRange(members, min, minIncl, max, maxIncl) + + for _, el := range members { + db.ssetRem(opts.Key, el) + } + c.WriteInt(len(members)) + }) +} + +// ZREMRANGEBYRANK +func (m *Miniredis) cmdZremrangebyrank(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + start int + end int + } + + opts.key = args[0] + if ok := optInt(c, args[1], &opts.start); !ok { + return + } + if ok := optInt(c, args[2], &opts.end); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + c.WriteInt(0) + return + } + + if db.t(opts.key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetMembers(opts.key) + rs, re := redisRange(len(members), opts.start, opts.end, false) + for _, el := range members[rs:re] { + db.ssetRem(opts.key, el) + } + c.WriteInt(re - rs) + }) +} + +// ZREMRANGEBYSCORE +func (m *Miniredis) cmdZremrangebyscore(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var ( + opts struct { + key string + min float64 + minIncl bool + max float64 + maxIncl bool + } + err error + ) + opts.key = args[0] + opts.min, opts.minIncl, err = parseFloatRange(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidMinMax) + return + } + opts.max, opts.maxIncl, err = parseFloatRange(args[2]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidMinMax) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + c.WriteInt(0) + return + } + + if db.t(opts.key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetElements(opts.key) + members = withSSRange(members, opts.min, opts.minIncl, opts.max, opts.maxIncl) + + for _, el := range members { + db.ssetRem(opts.key, el.member) + } + c.WriteInt(len(members)) + }) +} + +// ZSCORE +func (m *Miniredis) cmdZscore(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + key, member := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteNull() + return + } + + if db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + if !db.ssetExists(key, member) { + c.WriteNull() + return + } + + c.WriteFloat(db.ssetScore(key, member)) + }) +} + +// ZMSCORE +func (m *Miniredis) cmdZMscore(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + key, members := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteLen(len(members)) + for range members { + c.WriteNull() + } + return + } + + if db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + c.WriteLen(len(members)) + for _, member := range members { + if !db.ssetExists(key, member) { + c.WriteNull() + continue + } + c.WriteFloat(db.ssetScore(key, member)) + } + }) +} + +// parseFloatRange handles ZRANGEBYSCORE floats. They are inclusive unless the +// string starts with '(' +func parseFloatRange(s string) (float64, bool, error) { + if len(s) == 0 { + return 0, false, nil + } + inclusive := true + if s[0] == '(' { + s = s[1:] + inclusive = false + } + switch strings.ToLower(s) { + case "+inf": + return math.Inf(+1), true, nil + case "-inf": + return math.Inf(-1), true, nil + default: + f, err := strconv.ParseFloat(s, 64) + return f, inclusive, err + } +} + +// withSSRange limits a list of sorted set elements by the ZRANGEBYSCORE range +// logic. +func withSSRange(members ssElems, min float64, minIncl bool, max float64, maxIncl bool) ssElems { + gt := func(a, b float64) bool { return a > b } + gteq := func(a, b float64) bool { return a >= b } + + mincmp := gt + if minIncl { + mincmp = gteq + } + for i, m := range members { + if mincmp(m.score, min) { + members = members[i:] + goto checkmax + } + } + // all elements were smaller + return nil + +checkmax: + maxcmp := gteq + if maxIncl { + maxcmp = gt + } + for i, m := range members { + if maxcmp(m.score, max) { + members = members[:i] + break + } + } + + return members +} + +// withLexRange limits a list of sorted set elements. +func withLexRange(members []string, min string, minIncl bool, max string, maxIncl bool) []string { + if max == "-" || min == "+" { + return nil + } + if min != "-" { + found := false + if minIncl { + for i, m := range members { + if m >= min { + members = members[i:] + found = true + break + } + } + } else { + // Excluding min + for i, m := range members { + if m > min { + members = members[i:] + found = true + break + } + } + } + if !found { + return nil + } + } + if max != "+" { + if maxIncl { + for i, m := range members { + if m > max { + members = members[:i] + break + } + } + } else { + // Excluding max + for i, m := range members { + if m >= max { + members = members[:i] + break + } + } + } + } + return members +} + +// ZUNION +func (m *Miniredis) cmdZunion(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + numKeys, err := strconv.Atoi(args[0]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + args = args[1:] + if len(args) < numKeys { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + if numKeys <= 0 { + setDirty(c) + c.WriteError("ERR at least 1 input key is needed for ZUNION") + return + } + keys := args[:numKeys] + args = args[numKeys:] + + withScores := false + if len(args) > 0 && strings.ToUpper(args[len(args)-1]) == "WITHSCORES" { + withScores = true + args = args[:len(args)-1] + } + + opts := zunionOptions{ + Keys: keys, + WithWeights: false, + Weights: []float64{}, + Aggregate: "sum", + } + + if err := opts.parseArgs(args, numKeys); err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + sset, err := executeZUnion(db, opts) + if err != nil { + c.WriteError(err.Error()) + return + } + + if withScores { + c.WriteLen(len(sset) * 2) + } else { + c.WriteLen(len(sset)) + } + for _, el := range sset.byScore(asc) { + c.WriteBulk(el.member) + if withScores { + c.WriteFloat(el.score) + } + } + }) +} + +// ZUNIONSTORE +func (m *Miniredis) cmdZunionstore(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + destination := args[0] + numKeys, err := strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + args = args[2:] + if len(args) < numKeys { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + if numKeys <= 0 { + setDirty(c) + c.WriteError("ERR at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE") + return + } + keys := args[:numKeys] + args = args[numKeys:] + + opts := zunionOptions{ + Keys: keys, + WithWeights: false, + Weights: []float64{}, + Aggregate: "sum", + } + + if err := opts.parseArgs(args, numKeys); err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + deleteDest := true + for _, key := range keys { + if destination == key { + deleteDest = false + } + } + if deleteDest { + db.del(destination, true) + } + + sset, err := executeZUnion(db, opts) + if err != nil { + c.WriteError(err.Error()) + return + } + db.ssetSet(destination, sset) + c.WriteInt(sset.card()) + }) +} + +type zunionOptions struct { + Keys []string + WithWeights bool + Weights []float64 + Aggregate string +} + +func (opts *zunionOptions) parseArgs(args []string, numKeys int) error { + for len(args) > 0 { + switch strings.ToLower(args[0]) { + case "weights": + if len(args) < numKeys+1 { + return errors.New(msgSyntaxError) + } + for i := 0; i < numKeys; i++ { + f, err := strconv.ParseFloat(args[i+1], 64) + if err != nil { + return errors.New("ERR weight value is not a float") + } + opts.Weights = append(opts.Weights, f) + } + opts.WithWeights = true + args = args[numKeys+1:] + case "aggregate": + if len(args) < 2 { + return errors.New(msgSyntaxError) + } + opts.Aggregate = strings.ToLower(args[1]) + switch opts.Aggregate { + default: + return errors.New(msgSyntaxError) + case "sum", "min", "max": + } + args = args[2:] + default: + return errors.New(msgSyntaxError) + } + } + return nil +} + +func executeZUnion(db *RedisDB, opts zunionOptions) (sortedSet, error) { + sset := sortedSet{} + for i, key := range opts.Keys { + if !db.exists(key) { + continue + } + + var set map[string]float64 + switch db.t(key) { + case keyTypeSet: + set = map[string]float64{} + for elem := range db.setKeys[key] { + set[elem] = 1.0 + } + case keyTypeSortedSet: + set = db.sortedSet(key) + default: + return nil, errors.New(msgWrongType) + } + + for member, score := range set { + if opts.WithWeights { + score *= opts.Weights[i] + } + old, ok := sset[member] + if !ok { + sset[member] = score + continue + } + switch opts.Aggregate { + default: + panic("Invalid aggregate") + case "sum": + sset[member] += score + case "min": + if score < old { + sset[member] = score + } + case "max": + if score > old { + sset[member] = score + } + } + } + } + + return sset, nil +} + +// ZSCAN +func (m *Miniredis) cmdZscan(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + var opts struct { + key string + cursor int + count int + withMatch bool + match string + } + + opts.key = args[0] + if ok := optIntErr(c, args[1], &opts.cursor, msgInvalidCursor); !ok { + return + } + args = args[2:] + // MATCH and COUNT options + for len(args) > 0 { + if strings.ToLower(args[0]) == "count" { + if len(args) < 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + count, err := strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if count <= 0 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + opts.count = count + args = args[2:] + continue + } + if strings.ToLower(args[0]) == "match" { + if len(args) < 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + opts.withMatch = true + opts.match = args[1] + args = args[2:] + continue + } + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + if db.exists(opts.key) && db.t(opts.key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetMembers(opts.key) + if opts.withMatch { + members, _ = matchKeys(members, opts.match) + } + + low := opts.cursor + high := low + opts.count + // validate high is correct + if high > len(members) || high == 0 { + high = len(members) + } + if opts.cursor > high { + // invalid cursor + c.WriteLen(2) + c.WriteBulk("0") // no next cursor + c.WriteLen(0) // no elements + return + } + cursorValue := low + opts.count + if cursorValue >= len(members) { + cursorValue = 0 // no next cursor + } + members = members[low:high] + + c.WriteLen(2) + c.WriteBulk(fmt.Sprintf("%d", cursorValue)) + // HSCAN gives key, values. + c.WriteLen(len(members) * 2) + for _, k := range members { + c.WriteBulk(k) + c.WriteFloat(db.ssetScore(opts.key, k)) + } + }) +} + +// ZPOPMAX and ZPOPMIN +func (m *Miniredis) cmdZpopmax(reverse bool) server.Cmd { + return func(c *server.Peer, cmd string, args []string) { + if len(args) < 1 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + if !m.handleAuth(c) { + return + } + + key := args[0] + count := 1 + var err error + if len(args) > 1 { + count, err = strconv.Atoi(args[1]) + if err != nil || count < 0 { + setDirty(c) + c.WriteError(msgInvalidRange) + return + } + } + + withScores := true + if len(args) > 2 { + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteLen(0) + return + } + + if db.t(key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetMembers(key) + if reverse { + reverseSlice(members) + } + rs, re := redisRange(len(members), 0, count-1, false) + if withScores { + c.WriteLen((re - rs) * 2) + } else { + c.WriteLen(re - rs) + } + for _, el := range members[rs:re] { + c.WriteBulk(el) + if withScores { + c.WriteFloat(db.ssetScore(key, el)) + } + db.ssetRem(key, el) + } + }) + } +} + +// ZRANDMEMBER +func (m *Miniredis) cmdZrandmember(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + var opts struct { + key string + withCount bool + count int + withScores bool + } + + opts.key = args[0] + args = args[1:] + + if len(args) > 0 { + // can be negative + if ok := optInt(c, args[0], &opts.count); !ok { + return + } + opts.withCount = true + args = args[1:] + } + + if len(args) > 0 && strings.ToUpper(args[0]) == "WITHSCORES" { + opts.withScores = true + args = args[1:] + } + + if len(args) > 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + if opts.withCount { + c.WriteLen(0) + } else { + c.WriteNull() + } + return + } + + if db.t(opts.key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + if !opts.withCount { + member := db.ssetRandomMember(opts.key) + if member == "" { + c.WriteNull() + return + } + c.WriteBulk(member) + return + } + + var members []string + switch { + case opts.count == 0: + c.WriteStrings(nil) + return + case opts.count > 0: + allMembers := db.ssetMembers(opts.key) + db.master.shuffle(allMembers) + if len(allMembers) > opts.count { + allMembers = allMembers[:opts.count] + } + members = allMembers + case opts.count < 0: + for i := 0; i < -opts.count; i++ { + members = append(members, db.ssetRandomMember(opts.key)) + } + } + if opts.withScores { + c.WriteLen(len(members) * 2) + for _, m := range members { + c.WriteBulk(m) + c.WriteFloat(db.ssetScore(opts.key, m)) + } + return + } + c.WriteStrings(members) + }) +} + +type optsRange struct { + Key string + Min string + Max string + Reverse bool + WithScores bool +} + +func runRange(m *Miniredis, c *server.Peer, cctx *connCtx, opts optsRange) { + min, minErr := strconv.Atoi(opts.Min) + max, maxErr := strconv.Atoi(opts.Max) + if minErr != nil || maxErr != nil { + c.WriteError(msgInvalidInt) + return + } + + db := m.db(cctx.selectedDB) + + if !db.exists(opts.Key) { + c.WriteLen(0) + return + } + + if db.t(opts.Key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetMembers(opts.Key) + if opts.Reverse { + reverseSlice(members) + } + rs, re := redisRange(len(members), min, max, false) + if opts.WithScores { + c.WriteLen((re - rs) * 2) + } else { + c.WriteLen(re - rs) + } + for _, el := range members[rs:re] { + c.WriteBulk(el) + if opts.WithScores { + c.WriteFloat(db.ssetScore(opts.Key, el)) + } + } +} + +type optsRangeByScore struct { + Key string + Min string + Max string + Reverse bool + WithLimit bool + Offset string + Count string + WithScores bool +} + +func runRangeByScore(m *Miniredis, c *server.Peer, cctx *connCtx, opts optsRangeByScore) { + var limitOffset, limitCount int + var err error + if opts.WithLimit { + limitOffset, err = strconv.Atoi(opts.Offset) + if err != nil { + c.WriteError(msgInvalidInt) + return + } + limitCount, err = strconv.Atoi(opts.Count) + if err != nil { + c.WriteError(msgInvalidInt) + return + } + } + min, minIncl, minErr := parseFloatRange(opts.Min) + max, maxIncl, maxErr := parseFloatRange(opts.Max) + if minErr != nil || maxErr != nil { + c.WriteError(msgInvalidMinMax) + return + } + + db := m.db(cctx.selectedDB) + + if !db.exists(opts.Key) { + c.WriteLen(0) + return + } + + if db.t(opts.Key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetElements(opts.Key) + if opts.Reverse { + min, max = max, min + minIncl, maxIncl = maxIncl, minIncl + } + members = withSSRange(members, min, minIncl, max, maxIncl) + if opts.Reverse { + reverseElems(members) + } + + // Apply LIMIT ranges. That's . Unlike RANGE. + if opts.WithLimit { + if limitOffset < 0 { + members = ssElems{} + } else { + if limitOffset < len(members) { + members = members[limitOffset:] + } else { + // out of range + members = ssElems{} + } + if limitCount >= 0 { + if len(members) > limitCount { + members = members[:limitCount] + } + } + } + } + + if opts.WithScores { + c.WriteLen(len(members) * 2) + } else { + c.WriteLen(len(members)) + } + for _, el := range members { + c.WriteBulk(el.member) + if opts.WithScores { + c.WriteFloat(el.score) + } + } +} + +type optsRangeByLex struct { + Key string + Min string + Max string + Reverse bool + WithLimit bool + Offset string + Count string + WithScores bool +} + +func runRangeByLex(m *Miniredis, c *server.Peer, cctx *connCtx, opts optsRangeByLex) { + var limitOffset, limitCount int + var err error + if opts.WithLimit { + limitOffset, err = strconv.Atoi(opts.Offset) + if err != nil { + c.WriteError(msgInvalidInt) + return + } + limitCount, err = strconv.Atoi(opts.Count) + if err != nil { + c.WriteError(msgInvalidInt) + return + } + } + min, minIncl, minErr := parseLexrange(opts.Min) + max, maxIncl, maxErr := parseLexrange(opts.Max) + if minErr != nil || maxErr != nil { + c.WriteError(msgInvalidRangeItem) + return + } + + db := m.db(cctx.selectedDB) + + if !db.exists(opts.Key) { + c.WriteLen(0) + return + } + + if db.t(opts.Key) != keyTypeSortedSet { + c.WriteError(ErrWrongType.Error()) + return + } + + members := db.ssetMembers(opts.Key) + // Just key sort. If scores are not the same we don't care. + sort.Strings(members) + if opts.Reverse { + min, max = max, min + minIncl, maxIncl = maxIncl, minIncl + } + members = withLexRange(members, min, minIncl, max, maxIncl) + if opts.Reverse { + reverseSlice(members) + } + + // Apply LIMIT ranges. That's . Unlike RANGE. + if opts.WithLimit { + if limitOffset < 0 { + members = nil + } else { + if limitOffset < len(members) { + members = members[limitOffset:] + } else { + // out of range + members = nil + } + if limitCount >= 0 { + if len(members) > limitCount { + members = members[:limitCount] + } + } + } + } + + c.WriteLen(len(members)) + for _, el := range members { + c.WriteBulk(el) + } +} + +// optLexrange handles ZRANGE{,BYLEX} ranges. They start with '[', '(', or are +// '+' or '-'. +// Sets destValue and destInclusive. destValue can be '+' or '-'. +func parseLexrange(s string) (string, bool, error) { + if len(s) == 0 { + return "", false, errors.New(msgInvalidRangeItem) + } + + if s == "+" || s == "-" { + return s, false, nil + } + + switch s[0] { + case '(': + return s[1:], false, nil + case '[': + return s[1:], true, nil + default: + return "", false, errors.New(msgInvalidRangeItem) + } +} + +func sortedKeys(m map[string]float64) []string { + var keys []string + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_stream.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_stream.go new file mode 100644 index 00000000..e8d6570c --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_stream.go @@ -0,0 +1,1698 @@ +// Commands from https://redis.io/commands#stream + +package miniredis + +import ( + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsStream handles all stream operations. +func commandsStream(m *Miniredis) { + m.srv.Register("XADD", m.cmdXadd) + m.srv.Register("XLEN", m.cmdXlen, server.ReadOnlyOption()) + m.srv.Register("XREAD", m.cmdXread, server.ReadOnlyOption()) + m.srv.Register("XRANGE", m.makeCmdXrange(false), server.ReadOnlyOption()) + m.srv.Register("XREVRANGE", m.makeCmdXrange(true), server.ReadOnlyOption()) + m.srv.Register("XGROUP", m.cmdXgroup) + m.srv.Register("XINFO", m.cmdXinfo) + m.srv.Register("XREADGROUP", m.cmdXreadgroup) + m.srv.Register("XACK", m.cmdXack) + m.srv.Register("XDEL", m.cmdXdel) + m.srv.Register("XPENDING", m.cmdXpending, server.ReadOnlyOption()) + m.srv.Register("XTRIM", m.cmdXtrim) + m.srv.Register("XAUTOCLAIM", m.cmdXautoclaim) + m.srv.Register("XCLAIM", m.cmdXclaim) +} + +// XADD +func (m *Miniredis) cmdXadd(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(4)) { + return + } + + key, args := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + maxlen := -1 + minID := "" + makeStream := true + if strings.ToLower(args[0]) == "nomkstream" { + args = args[1:] + makeStream = false + } + if strings.ToLower(args[0]) == "maxlen" { + args = args[1:] + // ignore exact/approximate trimming modifiers + if args[0] == "~" || args[0] == "=" { + args = args[1:] + } + n, err := strconv.Atoi(args[0]) + if err != nil { + c.WriteError(msgInvalidInt) + return + } + if n < 0 { + c.WriteError("ERR The MAXLEN argument must be >= 0.") + return + } + maxlen = n + args = args[1:] + } else if strings.ToLower(args[0]) == "minid" { + args = args[1:] + // ignore exact/approximate trimming modifiers + if args[0] == "~" || args[0] == "=" { + args = args[1:] + } + minID = args[0] + args = args[1:] + } + if len(args) < 1 { + c.WriteError(errWrongNumber(cmd)) + return + } + entryID, args := args[0], args[1:] + + // args must be composed of field/value pairs. + if len(args) == 0 || len(args)%2 != 0 { + c.WriteError("ERR wrong number of arguments for XADD") // non-default message + return + } + + var values []string + for len(args) > 0 { + values = append(values, args[0], args[1]) + args = args[2:] + } + + db := m.db(ctx.selectedDB) + s, err := db.stream(key) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil { + if !makeStream { + c.WriteNull() + return + } + s, _ = db.newStream(key) + } + + newID, err := s.add(entryID, values, m.effectiveNow()) + if err != nil { + switch err { + case errInvalidEntryID: + c.WriteError(msgInvalidStreamID) + default: + c.WriteError(err.Error()) + } + return + } + if maxlen >= 0 { + s.trim(maxlen) + } + if minID != "" { + s.trimBefore(minID) + } + db.incr(key) + + c.WriteBulk(newID) + }) +} + +// XLEN +func (m *Miniredis) cmdXlen(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + s, err := db.stream(key) + if err != nil { + c.WriteError(err.Error()) + } + if s == nil { + // No such key. That's zero length. + c.WriteInt(0) + return + } + + c.WriteInt(len(s.entries)) + }) +} + +// XRANGE and XREVRANGE +func (m *Miniredis) makeCmdXrange(reverse bool) server.Cmd { + return func(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + if len(args) == 4 || len(args) > 5 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + opts := struct { + key string + startKey string + startExclusive bool + endKey string + endExclusive bool + }{ + key: args[0], + startKey: args[1], + endKey: args[2], + } + if strings.HasPrefix(opts.startKey, "(") { + opts.startExclusive = true + opts.startKey = opts.startKey[1:] + if opts.startKey == "-" || opts.startKey == "+" { + setDirty(c) + c.WriteError(msgInvalidStreamID) + return + } + } + if strings.HasPrefix(opts.endKey, "(") { + opts.endExclusive = true + opts.endKey = opts.endKey[1:] + if opts.endKey == "-" || opts.endKey == "+" { + setDirty(c) + c.WriteError(msgInvalidStreamID) + return + } + } + + countArg := "0" + if len(args) == 5 { + if strings.ToLower(args[3]) != "count" { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + countArg = args[4] + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + start, err := formatStreamRangeBound(opts.startKey, true, reverse) + if err != nil { + c.WriteError(msgInvalidStreamID) + return + } + end, err := formatStreamRangeBound(opts.endKey, false, reverse) + if err != nil { + c.WriteError(msgInvalidStreamID) + return + } + count, err := strconv.Atoi(countArg) + if err != nil { + c.WriteError(msgInvalidInt) + return + } + + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + c.WriteLen(0) + return + } + + if db.t(opts.key) != keyTypeStream { + c.WriteError(ErrWrongType.Error()) + return + } + + var entries = db.streamKeys[opts.key].entries + if reverse { + entries = reversedStreamEntries(entries) + } + if count == 0 { + count = len(entries) + } + + var returnedEntries []StreamEntry + for _, entry := range entries { + if len(returnedEntries) == count { + break + } + + if !reverse { + // Break if entry ID > end + if streamCmp(entry.ID, end) == 1 { + break + } + + // Continue if entry ID < start + if streamCmp(entry.ID, start) == -1 { + continue + } + } else { + // Break if entry iD < end + if streamCmp(entry.ID, end) == -1 { + break + } + + // Continue if entry ID > start. + if streamCmp(entry.ID, start) == 1 { + continue + } + } + + // Continue if start exclusive and entry ID == start + if opts.startExclusive && streamCmp(entry.ID, start) == 0 { + continue + } + // Continue if end exclusive and entry ID == end + if opts.endExclusive && streamCmp(entry.ID, end) == 0 { + continue + } + + returnedEntries = append(returnedEntries, entry) + } + + c.WriteLen(len(returnedEntries)) + for _, entry := range returnedEntries { + c.WriteLen(2) + c.WriteBulk(entry.ID) + c.WriteLen(len(entry.Values)) + for _, v := range entry.Values { + c.WriteBulk(v) + } + } + }) + } +} + +// XGROUP +func (m *Miniredis) cmdXgroup(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + subCmd, args := strings.ToLower(args[0]), args[1:] + switch subCmd { + case "create": + m.cmdXgroupCreate(c, cmd, args) + case "destroy": + m.cmdXgroupDestroy(c, cmd, args) + case "createconsumer": + m.cmdXgroupCreateconsumer(c, cmd, args) + case "delconsumer": + m.cmdXgroupDelconsumer(c, cmd, args) + case "help", + "setid": + err := fmt.Sprintf("ERR 'XGROUP %s' not supported", subCmd) + setDirty(c) + c.WriteError(err) + default: + setDirty(c) + c.WriteError(fmt.Sprintf( + "ERR unknown subcommand '%s'. Try XGROUP HELP.", + subCmd, + )) + } +} + +// XGROUP CREATE +func (m *Miniredis) cmdXgroupCreate(c *server.Peer, cmd string, args []string) { + if len(args) != 3 && len(args) != 4 { + setDirty(c) + c.WriteError(errWrongNumber("CREATE")) + return + } + stream, group, id := args[0], args[1], args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + s, err := db.stream(stream) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil && len(args) == 4 && strings.ToUpper(args[3]) == "MKSTREAM" { + if s, err = db.newStream(stream); err != nil { + c.WriteError(err.Error()) + return + } + } + if s == nil { + c.WriteError(msgXgroupKeyNotFound) + return + } + + if err := s.createGroup(group, id); err != nil { + c.WriteError(err.Error()) + return + } + + c.WriteOK() + }) +} + +// XGROUP DESTROY +func (m *Miniredis) cmdXgroupDestroy(c *server.Peer, cmd string, args []string) { + if len(args) != 2 { + setDirty(c) + c.WriteError(errWrongNumber("DESTROY")) + return + } + stream, groupName := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + s, err := db.stream(stream) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil { + c.WriteError(msgXgroupKeyNotFound) + return + } + + if _, ok := s.groups[groupName]; !ok { + c.WriteInt(0) + return + } + delete(s.groups, groupName) + c.WriteInt(1) + }) +} + +// XGROUP CREATECONSUMER +func (m *Miniredis) cmdXgroupCreateconsumer(c *server.Peer, cmd string, args []string) { + if len(args) != 3 { + setDirty(c) + c.WriteError(errWrongNumber("CREATECONSUMER")) + return + } + key, groupName, consumerName := args[0], args[1], args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + s, err := db.stream(key) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil { + c.WriteError(msgXgroupKeyNotFound) + return + } + + g, ok := s.groups[groupName] + if !ok { + err := fmt.Sprintf("NOGROUP No such consumer group '%s' for key name '%s'", groupName, key) + c.WriteError(err) + return + } + + if _, ok = g.consumers[consumerName]; ok { + c.WriteInt(0) + return + } + g.consumers[consumerName] = &consumer{} + c.WriteInt(1) + }) +} + +// XGROUP DELCONSUMER +func (m *Miniredis) cmdXgroupDelconsumer(c *server.Peer, cmd string, args []string) { + if len(args) != 3 { + setDirty(c) + c.WriteError(errWrongNumber("DELCONSUMER")) + return + } + key, groupName, consumerName := args[0], args[1], args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + s, err := db.stream(key) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil { + c.WriteError(msgXgroupKeyNotFound) + return + } + + g, ok := s.groups[groupName] + if !ok { + err := fmt.Sprintf("NOGROUP No such consumer group '%s' for key name '%s'", groupName, key) + c.WriteError(err) + return + } + + consumer, ok := g.consumers[consumerName] + if !ok { + c.WriteInt(0) + return + } + defer delete(g.consumers, consumerName) + + if consumer.numPendingEntries > 0 { + newPending := make([]pendingEntry, 0) + for _, entry := range g.pending { + if entry.consumer != consumerName { + newPending = append(newPending, entry) + } + } + g.pending = newPending + } + c.WriteInt(consumer.numPendingEntries) + }) +} + +// XINFO +func (m *Miniredis) cmdXinfo(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + subCmd, args := strings.ToUpper(args[0]), args[1:] + switch subCmd { + case "STREAM": + m.cmdXinfoStream(c, args) + case "CONSUMERS": + m.cmdXinfoConsumers(c, args) + case "GROUPS": + m.cmdXinfoGroups(c, args) + case "HELP": + err := fmt.Sprintf("'XINFO %s' not supported", strings.Join(args, " ")) + setDirty(c) + c.WriteError(err) + default: + setDirty(c) + c.WriteError(fmt.Sprintf( + "ERR unknown subcommand or wrong number of arguments for '%s'. Try XINFO HELP.", + subCmd, + )) + } +} + +// XINFO STREAM +// Produces only part of full command output +func (m *Miniredis) cmdXinfoStream(c *server.Peer, args []string) { + if len(args) < 1 { + setDirty(c) + c.WriteError(errWrongNumber("STREAM")) + return + } + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + s, err := db.stream(key) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil { + c.WriteError(msgKeyNotFound) + return + } + + c.WriteMapLen(2) + c.WriteBulk("length") + c.WriteInt(len(s.entries)) + c.WriteBulk("last-generated-id") + c.WriteBulk(s.lastAllocatedID) + }) +} + +// XINFO GROUPS +func (m *Miniredis) cmdXinfoGroups(c *server.Peer, args []string) { + if len(args) != 1 { + setDirty(c) + c.WriteError(errWrongNumber("GROUPS")) + return + } + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + s, err := db.stream(key) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil { + c.WriteError(msgKeyNotFound) + return + } + + c.WriteLen(len(s.groups)) + for name, g := range s.groups { + c.WriteMapLen(6) + + c.WriteBulk("name") + c.WriteBulk(name) + c.WriteBulk("consumers") + c.WriteInt(len(g.consumers)) + c.WriteBulk("pending") + c.WriteInt(len(g.activePending())) + c.WriteBulk("last-delivered-id") + c.WriteBulk(g.lastID) + c.WriteBulk("entries-read") + c.WriteNull() + c.WriteBulk("lag") + c.WriteInt(len(g.stream.entries)) + } + }) +} + +// XINFO CONSUMERS +// Please note that this is only a partial implementation, for it does not +// return each consumer's "idle" value, which indicates "the number of +// milliseconds that have passed since the consumer last interacted with the +// server." +func (m *Miniredis) cmdXinfoConsumers(c *server.Peer, args []string) { + if len(args) != 2 { + setDirty(c) + c.WriteError(errWrongNumber("CONSUMERS")) + return + } + key, groupName := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + s, err := db.stream(key) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil { + c.WriteError(msgKeyNotFound) + return + } + + g, ok := s.groups[groupName] + if !ok { + err := fmt.Sprintf("NOGROUP No such consumer group '%s' for key name '%s'", groupName, key) + c.WriteError(err) + return + } + + var consumerNames []string + for name := range g.consumers { + consumerNames = append(consumerNames, name) + } + sort.Strings(consumerNames) + + c.WriteLen(len(consumerNames)) + for _, name := range consumerNames { + cons := g.consumers[name] + + c.WriteMapLen(4) + c.WriteBulk("name") + c.WriteBulk(name) + c.WriteBulk("pending") + c.WriteInt(cons.numPendingEntries) + // TODO: these times aren't set for all commands + c.WriteBulk("idle") + c.WriteInt(m.sinceMilli(cons.lastSeen)) + c.WriteBulk("inactive") + c.WriteInt(m.sinceMilli(cons.lastSuccess)) + } + }) +} + +func (m *Miniredis) sinceMilli(t time.Time) int { + if t.IsZero() { + return -1 + } + return int(m.effectiveNow().Sub(t).Milliseconds()) +} + +// XREADGROUP +func (m *Miniredis) cmdXreadgroup(c *server.Peer, cmd string, args []string) { + // XREADGROUP GROUP group consumer STREAMS key ID + if !m.isValidCMD(c, cmd, args, atLeast(6)) { + return + } + + var opts struct { + group string + consumer string + count int + noack bool + streams []string + ids []string + block bool + blockTimeout time.Duration + } + + if strings.ToUpper(args[0]) != "GROUP" { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + opts.group, opts.consumer, args = args[1], args[2], args[3:] + + var err error +parsing: + for len(args) > 0 { + switch strings.ToUpper(args[0]) { + case "COUNT": + if len(args) < 2 { + err = errors.New(errWrongNumber(cmd)) + break parsing + } + + opts.count, err = strconv.Atoi(args[1]) + if err != nil { + break parsing + } + + args = args[2:] + case "BLOCK": + err = parseBlock(cmd, args, &opts.block, &opts.blockTimeout) + if err != nil { + break parsing + } + args = args[2:] + case "NOACK": + args = args[1:] + opts.noack = true + case "STREAMS": + args = args[1:] + + if len(args)%2 != 0 { + err = errors.New(msgXreadUnbalanced) + break parsing + } + + opts.streams, opts.ids = args[0:len(args)/2], args[len(args)/2:] + break parsing + default: + err = fmt.Errorf("ERR incorrect argument %s", args[0]) + break parsing + } + } + + if err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + + if len(opts.streams) == 0 || len(opts.ids) == 0 { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return + } + + for _, id := range opts.ids { + if id != `>` { + opts.block = false + } + } + + if !opts.block { + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + res, err := xreadgroup( + db, + opts.group, + opts.consumer, + opts.noack, + opts.streams, + opts.ids, + opts.count, + m.effectiveNow(), + ) + if err != nil { + c.WriteError(err.Error()) + return + } + writeXread(c, opts.streams, res) + }) + return + } + + blocking( + m, + c, + opts.blockTimeout, + func(c *server.Peer, ctx *connCtx) bool { + db := m.db(ctx.selectedDB) + res, err := xreadgroup( + db, + opts.group, + opts.consumer, + opts.noack, + opts.streams, + opts.ids, + opts.count, + m.effectiveNow(), + ) + if err != nil { + c.WriteError(err.Error()) + return true + } + if len(res) == 0 { + return false + } + writeXread(c, opts.streams, res) + return true + }, + func(c *server.Peer) { // timeout + c.WriteLen(-1) + }, + ) +} + +func xreadgroup( + db *RedisDB, + group, + consumer string, + noack bool, + streams []string, + ids []string, + count int, + now time.Time, +) (map[string][]StreamEntry, error) { + res := map[string][]StreamEntry{} + for i, key := range streams { + id := ids[i] + + g, err := db.streamGroup(key, group) + if err != nil { + return nil, err + } + if g == nil { + return nil, errXreadgroup(key, group) + } + + if _, err := parseStreamID(id); id != `>` && err != nil { + return nil, err + } + entries := g.readGroup(now, consumer, id, count, noack) + if id == `>` && len(entries) == 0 { + continue + } + + res[key] = entries + } + return res, nil +} + +// XACK +func (m *Miniredis) cmdXack(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + key, group, ids := args[0], args[1], args[2:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + g, err := db.streamGroup(key, group) + if err != nil { + c.WriteError(err.Error()) + return + } + if g == nil { + c.WriteInt(0) + return + } + + cnt, err := g.ack(ids) + if err != nil { + c.WriteError(err.Error()) + return + } + c.WriteInt(cnt) + }) +} + +// XDEL +func (m *Miniredis) cmdXdel(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + stream, ids := args[0], args[1:] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + s, err := db.stream(stream) + if err != nil { + c.WriteError(err.Error()) + return + } + if s == nil { + c.WriteInt(0) + return + } + + n, err := s.delete(ids) + if err != nil { + c.WriteError(err.Error()) + return + } + db.incr(stream) + c.WriteInt(n) + }) +} + +// XREAD +func (m *Miniredis) cmdXread(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + var ( + opts struct { + count int + streams []string + ids []string + block bool + blockTimeout time.Duration + } + err error + ) + +parsing: + for len(args) > 0 { + switch strings.ToUpper(args[0]) { + case "COUNT": + if len(args) < 2 { + err = errors.New(errWrongNumber(cmd)) + break parsing + } + + opts.count, err = strconv.Atoi(args[1]) + if err != nil { + break parsing + } + args = args[2:] + case "BLOCK": + err = parseBlock(cmd, args, &opts.block, &opts.blockTimeout) + if err != nil { + break parsing + } + args = args[2:] + case "STREAMS": + args = args[1:] + + if len(args)%2 != 0 { + err = errors.New(msgXreadUnbalanced) + break parsing + } + + opts.streams, opts.ids = args[0:len(args)/2], args[len(args)/2:] + for i, id := range opts.ids { + if _, err := parseStreamID(id); id != `$` && err != nil { + setDirty(c) + c.WriteError(msgInvalidStreamID) + return + } else if id == "$" { + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(getCtx(c).selectedDB) + stream, ok := db.streamKeys[opts.streams[i]] + if ok { + opts.ids[i] = stream.lastID() + } else { + opts.ids[i] = "0-0" + } + }) + } + } + args = nil + break parsing + default: + err = fmt.Errorf("ERR incorrect argument %s", args[0]) + break parsing + } + } + if err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + + if !opts.block { + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + res := xread(db, opts.streams, opts.ids, opts.count) + writeXread(c, opts.streams, res) + }) + return + } + blocking( + m, + c, + opts.blockTimeout, + func(c *server.Peer, ctx *connCtx) bool { + db := m.db(ctx.selectedDB) + res := xread(db, opts.streams, opts.ids, opts.count) + if len(res) == 0 { + return false + } + writeXread(c, opts.streams, res) + return true + }, + func(c *server.Peer) { // timeout + c.WriteLen(-1) + }, + ) +} + +func xread(db *RedisDB, streams []string, ids []string, count int) map[string][]StreamEntry { + res := map[string][]StreamEntry{} + for i := range streams { + stream := streams[i] + id := ids[i] + + var s, ok = db.streamKeys[stream] + if !ok { + continue + } + entries := s.entries + if len(entries) == 0 { + continue + } + + entryCount := count + if entryCount == 0 { + entryCount = len(entries) + } + + var returnedEntries []StreamEntry + for _, entry := range entries { + if len(returnedEntries) == entryCount { + break + } + if id == "$" { + id = s.lastID() + } + if streamCmp(entry.ID, id) <= 0 { + continue + } + returnedEntries = append(returnedEntries, entry) + } + if len(returnedEntries) > 0 { + res[stream] = returnedEntries + } + } + return res +} + +func writeXread(c *server.Peer, streams []string, res map[string][]StreamEntry) { + if len(res) == 0 { + c.WriteLen(-1) + return + } + c.WriteLen(len(res)) + for _, stream := range streams { + entries, ok := res[stream] + if !ok { + continue + } + c.WriteLen(2) + c.WriteBulk(stream) + c.WriteLen(len(entries)) + for _, entry := range entries { + c.WriteLen(2) + c.WriteBulk(entry.ID) + c.WriteLen(len(entry.Values)) + for _, v := range entry.Values { + c.WriteBulk(v) + } + } + } +} + +// XPENDING +func (m *Miniredis) cmdXpending(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + var opts struct { + key string + group string + summary bool + idle time.Duration + start, end string + count int + consumer *string + } + + opts.key, opts.group, args = args[0], args[1], args[2:] + opts.summary = true + if len(args) >= 3 { + opts.summary = false + + if strings.ToUpper(args[0]) == "IDLE" { + idleMs, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + opts.idle = time.Duration(idleMs) * time.Millisecond + + args = args[2:] + if len(args) < 3 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + } + + var err error + opts.start, err = formatStreamRangeBound(args[0], true, false) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidStreamID) + return + } + opts.end, err = formatStreamRangeBound(args[1], false, false) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidStreamID) + return + } + opts.count, err = strconv.Atoi(args[2]) // negative is allowed + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + args = args[3:] + + if len(args) == 1 { + opts.consumer, args = &args[0], args[1:] + } + } + if len(args) != 0 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + g, err := db.streamGroup(opts.key, opts.group) + if err != nil { + c.WriteError(err.Error()) + return + } + if g == nil { + c.WriteError(errReadgroup(opts.key, opts.group).Error()) + return + } + + if opts.summary { + writeXpendingSummary(c, *g) + return + } + writeXpending(m.effectiveNow(), c, *g, opts.idle, opts.start, opts.end, opts.count, opts.consumer) + }) +} + +func writeXpendingSummary(c *server.Peer, g streamGroup) { + pend := g.activePending() + if len(pend) == 0 { + c.WriteLen(4) + c.WriteInt(0) + c.WriteNull() + c.WriteNull() + c.WriteLen(-1) + return + } + + // format: + // - number of pending + // - smallest ID + // - highest ID + // - all consumers with > 0 pending items + c.WriteLen(4) + c.WriteInt(len(pend)) + c.WriteBulk(pend[0].id) + c.WriteBulk(pend[len(pend)-1].id) + cons := map[string]int{} + for id := range g.consumers { + cnt := g.pendingCount(id) + if cnt > 0 { + cons[id] = cnt + } + } + c.WriteLen(len(cons)) + var ids []string + for id := range cons { + ids = append(ids, id) + } + sort.Strings(ids) // be predicatable + for _, id := range ids { + c.WriteLen(2) + c.WriteBulk(id) + c.WriteBulk(strconv.Itoa(cons[id])) + } +} + +func writeXpending( + now time.Time, + c *server.Peer, + g streamGroup, + idle time.Duration, + start, + end string, + count int, + consumer *string, +) { + if len(g.pending) == 0 || count < 0 { + c.WriteLen(0) + return + } + + // format, list of: + // - message ID + // - consumer + // - milliseconds since delivery + // - delivery count + type entry struct { + id string + consumer string + millis int + count int + } + var res []entry + for _, p := range g.pending { + if len(res) >= count { + break + } + if consumer != nil && p.consumer != *consumer { + continue + } + if streamCmp(p.id, start) < 0 { + continue + } + if streamCmp(p.id, end) > 0 { + continue + } + timeSinceLastDelivery := now.Sub(p.lastDelivery) + if timeSinceLastDelivery >= idle { + res = append(res, entry{ + id: p.id, + consumer: p.consumer, + millis: int(timeSinceLastDelivery.Milliseconds()), + count: p.deliveryCount, + }) + } + } + c.WriteLen(len(res)) + for _, e := range res { + c.WriteLen(4) + c.WriteBulk(e.id) + c.WriteBulk(e.consumer) + c.WriteInt(e.millis) + c.WriteInt(e.count) + } +} + +// XTRIM +func (m *Miniredis) cmdXtrim(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + var opts struct { + stream string + strategy string + maxLen int // for MAXLEN + threshold string // for MINID + withLimit bool // "LIMIT" + withExact bool // "=" + withNearly bool // "~" + } + + opts.stream, opts.strategy, args = args[0], strings.ToUpper(args[1]), args[2:] + + if opts.strategy != "MAXLEN" && opts.strategy != "MINID" { + setDirty(c) + c.WriteError(msgXtrimInvalidStrategy) + return + } + + // Ignore nearly exact trimming parameters. + switch args[0] { + case "=": + opts.withExact = true + args = args[1:] + case "~": + opts.withNearly = true + args = args[1:] + } + + switch opts.strategy { + case "MAXLEN": + maxLen, err := strconv.Atoi(args[0]) + if err != nil { + setDirty(c) + c.WriteError(msgXtrimInvalidMaxLen) + return + } + opts.maxLen = maxLen + case "MINID": + opts.threshold = args[0] + } + args = args[1:] + + if len(args) == 2 && strings.ToUpper(args[0]) == "LIMIT" { + // Ignore LIMIT. + opts.withLimit = true + if _, err := strconv.Atoi(args[1]); err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + + args = args[2:] + } + + if len(args) != 0 { + setDirty(c) + c.WriteError(fmt.Sprintf("ERR incorrect argument %s", args[0])) + return + } + + if opts.withLimit && !opts.withNearly { + setDirty(c) + c.WriteError(fmt.Sprintf(msgXtrimInvalidLimit)) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + s, err := db.stream(opts.stream) + if err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + if s == nil { + c.WriteInt(0) + return + } + + switch opts.strategy { + case "MAXLEN": + entriesBefore := len(s.entries) + s.trim(opts.maxLen) + c.WriteInt(entriesBefore - len(s.entries)) + case "MINID": + n := s.trimBefore(opts.threshold) + c.WriteInt(n) + } + }) +} + +// XAUTOCLAIM +func (m *Miniredis) cmdXautoclaim(c *server.Peer, cmd string, args []string) { + // XAUTOCLAIM key group consumer min-idle-time start + if !m.isValidCMD(c, cmd, args, atLeast(5)) { + return + } + + var opts struct { + key string + group string + consumer string + minIdleTime time.Duration + start string + justId bool + count int + } + + opts.key, opts.group, opts.consumer = args[0], args[1], args[2] + n, err := strconv.Atoi(args[3]) + if err != nil { + setDirty(c) + c.WriteError("ERR Invalid min-idle-time argument for XAUTOCLAIM") + return + } + opts.minIdleTime = time.Millisecond * time.Duration(n) + + start_, err := formatStreamRangeBound(args[4], true, false) + if err != nil { + c.WriteError(msgInvalidStreamID) + return + } + opts.start = start_ + + args = args[5:] + + opts.count = 100 +parsing: + for len(args) > 0 { + switch strings.ToUpper(args[0]) { + case "COUNT": + if len(args) < 2 { + err = errors.New(errWrongNumber(cmd)) + break parsing + } + + opts.count, err = strconv.Atoi(args[1]) + if err != nil { + break parsing + } + + args = args[2:] + case "JUSTID": + args = args[1:] + opts.justId = true + default: + err = errors.New(msgSyntaxError) + break parsing + } + } + + if err != nil { + setDirty(c) + c.WriteError(err.Error()) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + g, err := db.streamGroup(opts.key, opts.group) + if err != nil { + c.WriteError(err.Error()) + return + } + if g == nil { + c.WriteError(errReadgroup(opts.key, opts.group).Error()) + return + } + + nextCallId, entries := xautoclaim(m.effectiveNow(), *g, opts.minIdleTime, opts.start, opts.count, opts.consumer) + writeXautoclaim(c, nextCallId, entries, opts.justId) + }) +} + +func xautoclaim( + now time.Time, + g streamGroup, + minIdleTime time.Duration, + start string, + count int, + consumerID string, +) (string, []StreamEntry) { + nextCallId := "0-0" + if len(g.pending) == 0 || count < 0 { + return nextCallId, nil + } + + msgs := g.pendingAfterOrEqual(start) + var res []StreamEntry + for i, p := range msgs { + if minIdleTime > 0 && now.Before(p.lastDelivery.Add(minIdleTime)) { + continue + } + + prevConsumerID := p.consumer + if _, ok := g.consumers[consumerID]; !ok { + g.consumers[consumerID] = &consumer{} + } + p.consumer = consumerID + + _, entry := g.stream.get(p.id) + // not found. Weird? + if entry == nil { + // TODO: support third element of return from XAUTOCLAIM, which + // should delete entries not found in the PEL during XAUTOCLAIM. + // (Introduced in Redis 7.0) + continue + } + + p.deliveryCount += 1 + p.lastDelivery = now + + g.consumers[prevConsumerID].numPendingEntries-- + g.consumers[consumerID].numPendingEntries++ + + msgs[i] = p + res = append(res, *entry) + + if len(res) >= count { + if len(msgs) > i+1 { + nextCallId = msgs[i+1].id + } + break + } + } + return nextCallId, res +} + +func writeXautoclaim(c *server.Peer, nextCallId string, res []StreamEntry, justId bool) { + c.WriteLen(3) + c.WriteBulk(nextCallId) + c.WriteLen(len(res)) + for _, entry := range res { + if justId { + c.WriteBulk(entry.ID) + continue + } + + c.WriteLen(2) + c.WriteBulk(entry.ID) + c.WriteLen(len(entry.Values)) + for _, v := range entry.Values { + c.WriteBulk(v) + } + } + // TODO: see "Redis 7" note + c.WriteLen(0) +} + +// XCLAIM +func (m *Miniredis) cmdXclaim(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(5)) { + return + } + + var opts struct { + key string + groupName string + consumerName string + minIdleTime time.Duration + newLastDelivery time.Time + ids []string + retryCount *int + force bool + justId bool + } + + opts.key, opts.groupName, opts.consumerName = args[0], args[1], args[2] + + minIdleTimeMillis, err := strconv.Atoi(args[3]) + if err != nil { + setDirty(c) + c.WriteError("ERR Invalid min-idle-time argument for XCLAIM") + return + } + opts.minIdleTime = time.Millisecond * time.Duration(minIdleTimeMillis) + + opts.newLastDelivery = m.effectiveNow() + opts.ids = append(opts.ids, args[4]) + + args = args[5:] + for len(args) > 0 { + arg := strings.ToUpper(args[0]) + if arg == "IDLE" || + arg == "TIME" || + arg == "RETRYCOUNT" || + arg == "FORCE" || + arg == "JUSTID" { + break + } + opts.ids = append(opts.ids, arg) + args = args[1:] + } + + for len(args) > 0 { + arg := strings.ToUpper(args[0]) + switch arg { + case "IDLE": + idleMs, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + setDirty(c) + c.WriteError("ERR Invalid IDLE option argument for XCLAIM") + return + } + if idleMs < 0 { + idleMs = 0 + } + opts.newLastDelivery = m.effectiveNow().Add(time.Millisecond * time.Duration(-idleMs)) + args = args[2:] + case "TIME": + timeMs, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + setDirty(c) + c.WriteError("ERR Invalid TIME option argument for XCLAIM") + return + } + opts.newLastDelivery = time.UnixMilli(timeMs) + args = args[2:] + case "RETRYCOUNT": + retryCount, err := strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError("ERR Invalid RETRYCOUNT option argument for XCLAIM") + return + } + opts.retryCount = &retryCount + args = args[2:] + case "FORCE": + opts.force = true + args = args[1:] + case "JUSTID": + opts.justId = true + args = args[1:] + default: + setDirty(c) + c.WriteError(fmt.Sprintf("ERR Unrecognized XCLAIM option '%s'", args[0])) + return + } + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + g, err := db.streamGroup(opts.key, opts.groupName) + if err != nil { + c.WriteError(err.Error()) + return + } + if g == nil { + c.WriteError(errReadgroup(opts.key, opts.groupName).Error()) + return + } + + claimedEntryIDs := m.xclaim(g, opts.consumerName, opts.minIdleTime, opts.newLastDelivery, opts.ids, opts.retryCount, opts.force) + writeXclaim(c, g.stream, claimedEntryIDs, opts.justId) + }) +} + +func (m *Miniredis) xclaim( + group *streamGroup, + consumerName string, + minIdleTime time.Duration, + newLastDelivery time.Time, + ids []string, + retryCount *int, + force bool, +) (claimedEntryIDs []string) { + for _, id := range ids { + pelPos, pelEntry := group.searchPending(id) + if pelEntry == nil { + group.setLastSeen(consumerName, m.effectiveNow()) + if !force { + continue + } + + if pelPos < len(group.pending) { + group.pending = append(group.pending[:pelPos+1], group.pending[pelPos:]...) + } else { + group.pending = append(group.pending, pendingEntry{}) + } + pelEntry = &group.pending[pelPos] + + *pelEntry = pendingEntry{ + id: id, + consumer: consumerName, + deliveryCount: 1, + } + group.setLastSuccess(consumerName, m.effectiveNow()) + } else { + group.consumers[pelEntry.consumer].numPendingEntries-- + pelEntry.consumer = consumerName + } + + if retryCount != nil { + pelEntry.deliveryCount = *retryCount + } else { + pelEntry.deliveryCount++ + } + pelEntry.lastDelivery = newLastDelivery + + // redis7: don't report entries which are deleted by now + if _, e := group.stream.get(id); e == nil { + continue + } + + claimedEntryIDs = append(claimedEntryIDs, id) + } + if len(claimedEntryIDs) == 0 { + group.setLastSeen(consumerName, m.effectiveNow()) + return + } + + if _, ok := group.consumers[consumerName]; !ok { + group.consumers[consumerName] = &consumer{} + } + consumer := group.consumers[consumerName] + consumer.numPendingEntries += len(claimedEntryIDs) + + group.setLastSuccess(consumerName, m.effectiveNow()) + return +} + +func writeXclaim(c *server.Peer, stream *streamKey, claimedEntryIDs []string, justId bool) { + c.WriteLen(len(claimedEntryIDs)) + for _, id := range claimedEntryIDs { + if justId { + c.WriteBulk(id) + continue + } + + _, entry := stream.get(id) + if entry == nil { + c.WriteNull() + continue + } + + c.WriteLen(2) + c.WriteBulk(entry.ID) + c.WriteStrings(entry.Values) + } +} + +func parseBlock(cmd string, args []string, block *bool, timeout *time.Duration) error { + if len(args) < 2 { + return errors.New(errWrongNumber(cmd)) + } + (*block) = true + ms, err := strconv.Atoi(args[1]) + if err != nil { + return errors.New(msgInvalidInt) + } + if ms < 0 { + return errors.New("ERR timeout is negative") + } + (*timeout) = time.Millisecond * time.Duration(ms) + return nil +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_string.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_string.go new file mode 100644 index 00000000..2c2ba8e0 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/cmd_string.go @@ -0,0 +1,1243 @@ +// Commands from https://redis.io/commands#string + +package miniredis + +import ( + "math/big" + "math/bits" + "strconv" + "strings" + "time" + + "github.com/alicebob/miniredis/v2/server" +) + +// commandsString handles all string value operations. +func commandsString(m *Miniredis) { + m.srv.Register("APPEND", m.cmdAppend) + m.srv.Register("BITCOUNT", m.cmdBitcount, server.ReadOnlyOption()) + m.srv.Register("BITOP", m.cmdBitop) + m.srv.Register("BITPOS", m.cmdBitpos, server.ReadOnlyOption()) + m.srv.Register("DECRBY", m.cmdDecrby) + m.srv.Register("DECR", m.cmdDecr) + m.srv.Register("DELEX", m.cmdDelex) + m.srv.Register("GETBIT", m.cmdGetbit, server.ReadOnlyOption()) + m.srv.Register("GETDEL", m.cmdGetdel) + m.srv.Register("GETEX", m.cmdGetex) + m.srv.Register("GET", m.cmdGet, server.ReadOnlyOption()) + m.srv.Register("GETRANGE", m.cmdGetrange, server.ReadOnlyOption()) + m.srv.Register("GETSET", m.cmdGetset) + m.srv.Register("INCRBYFLOAT", m.cmdIncrbyfloat) + m.srv.Register("INCRBY", m.cmdIncrby) + m.srv.Register("INCR", m.cmdIncr) + m.srv.Register("MGET", m.cmdMget, server.ReadOnlyOption()) + m.srv.Register("MSET", m.cmdMset) + m.srv.Register("MSETNX", m.cmdMsetnx) + m.srv.Register("PSETEX", m.cmdPsetex) + m.srv.Register("SETBIT", m.cmdSetbit) + m.srv.Register("SETEX", m.cmdSetex) + m.srv.Register("SET", m.cmdSet) + m.srv.Register("SETNX", m.cmdSetnx) + m.srv.Register("SETRANGE", m.cmdSetrange) + m.srv.Register("STRLEN", m.cmdStrlen, server.ReadOnlyOption()) +} + +// SET +func (m *Miniredis) cmdSet(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + var opts struct { + key string + value string + nx bool // set iff not exists + xx bool // set iff exists + keepttl bool // set keepttl + ttlSet bool + ttl time.Duration + get bool + } + + opts.key, opts.value, args = args[0], args[1], args[2:] + for len(args) > 0 { + timeUnit := time.Second + switch arg := strings.ToUpper(args[0]); arg { + case "NX": + opts.nx = true + args = args[1:] + continue + case "XX": + opts.xx = true + args = args[1:] + continue + case "KEEPTTL": + opts.keepttl = true + args = args[1:] + continue + case "PX", "PXAT": + timeUnit = time.Millisecond + fallthrough + case "EX", "EXAT": + if len(args) < 2 { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if opts.ttlSet { + // multiple ex/exat/px/pxat options set + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + expire, err := strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if expire <= 0 { + setDirty(c) + c.WriteError(msgInvalidSETime) + return + } + + if arg == "PXAT" || arg == "EXAT" { + opts.ttl = m.at(expire, timeUnit) + } else { + opts.ttl = time.Duration(expire) * timeUnit + } + opts.ttlSet = true + + args = args[2:] + continue + case "GET": + opts.get = true + args = args[1:] + continue + default: + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + readonly := false + if opts.nx { + if db.exists(opts.key) { + if opts.get { + // special case for SET NX GET + readonly = true + } else { + c.WriteNull() + return + } + } + } + if opts.xx { + if !db.exists(opts.key) { + if opts.get { + // special case for SET XX GET + readonly = true + } else { + c.WriteNull() + return + } + } + } + if opts.keepttl { + if val, ok := db.ttl[opts.key]; ok { + opts.ttl = val + } + } + if opts.get { + if t, ok := db.keys[opts.key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + } + + old, existed := db.stringKeys[opts.key] + if !readonly { + db.del(opts.key, true) // be sure to remove existing values of other type keys. + // a vanilla SET clears the expire + if opts.ttl >= 0 { // EXAT/PXAT can expire right away + db.stringSet(opts.key, opts.value) + } + if opts.ttl != 0 { + db.ttl[opts.key] = opts.ttl + } + } + if opts.get { + if !existed { + c.WriteNull() + } else { + c.WriteBulk(old) + } + return + } + c.WriteOK() + }) +} + +// SETEX +func (m *Miniredis) cmdSetex(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + key := args[0] + ttl, err := strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if ttl <= 0 { + setDirty(c) + c.WriteError(msgInvalidSETEXTime) + return + } + value := args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + db.del(key, true) // Clear any existing keys. + db.stringSet(key, value) + db.ttl[key] = time.Duration(ttl) * time.Second + c.WriteOK() + }) +} + +// PSETEX +func (m *Miniredis) cmdPsetex(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + ttl int + value string + } + + opts.key = args[0] + if ok := optInt(c, args[1], &opts.ttl); !ok { + return + } + if opts.ttl <= 0 { + setDirty(c) + c.WriteError(msgInvalidPSETEXTime) + return + } + opts.value = args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + db.del(opts.key, true) // Clear any existing keys. + db.stringSet(opts.key, opts.value) + db.ttl[opts.key] = time.Duration(opts.ttl) * time.Millisecond + c.WriteOK() + }) +} + +// SETNX +func (m *Miniredis) cmdSetnx(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + key, value := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if _, ok := db.keys[key]; ok { + c.WriteInt(0) + return + } + + db.stringSet(key, value) + c.WriteInt(1) + }) +} + +// MSET +func (m *Miniredis) cmdMset(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + if len(args)%2 != 0 { + setDirty(c) + // non-default error message + c.WriteError("ERR wrong number of arguments for MSET") + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + for len(args) > 0 { + key, value := args[0], args[1] + args = args[2:] + + db.del(key, true) // clear TTL + db.stringSet(key, value) + } + c.WriteOK() + }) +} + +// MSETNX +func (m *Miniredis) cmdMsetnx(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(2)) { + return + } + + if len(args)%2 != 0 { + setDirty(c) + // non-default error message (yes, with 'MSET'). + c.WriteError("ERR wrong number of arguments for MSET") + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + keys := map[string]string{} + existing := false + for len(args) > 0 { + key := args[0] + value := args[1] + args = args[2:] + keys[key] = value + if _, ok := db.keys[key]; ok { + existing = true + } + } + + res := 0 + if !existing { + res = 1 + for k, v := range keys { + // Nothing to delete. That's the whole point. + db.stringSet(k, v) + } + } + c.WriteInt(res) + }) +} + +// GET +func (m *Miniredis) cmdGet(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(key) { + c.WriteNull() + return + } + if db.t(key) != keyTypeString { + c.WriteError(msgWrongType) + return + } + + c.WriteBulk(db.stringGet(key)) + }) +} + +// GETEX +func (m *Miniredis) cmdGetex(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + var opts struct { + key string + ttl time.Duration + persist bool // remove existing TTL on the key. + } + + opts.key, args = args[0], args[1:] + if len(args) > 0 { + timeUnit := time.Second + switch arg := strings.ToUpper(args[0]); arg { + case "PERSIST": + if len(args) > 1 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + opts.persist = true + case "PX", "PXAT": + timeUnit = time.Millisecond + fallthrough + case "EX", "EXAT": + if len(args) != 2 { + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + expire, err := strconv.Atoi(args[1]) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidInt) + return + } + if expire <= 0 { + setDirty(c) + c.WriteError(msgInvalidSETime) + return + } + + if arg == "PXAT" || arg == "EXAT" { + opts.ttl = m.at(expire, timeUnit) + } else { + opts.ttl = time.Duration(expire) * timeUnit + } + default: + setDirty(c) + c.WriteError(msgSyntaxError) + return + } + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + c.WriteNull() + return + } + switch { + case opts.persist: + delete(db.ttl, opts.key) + case opts.ttl != 0: + db.ttl[opts.key] = opts.ttl + } + + if db.t(opts.key) != keyTypeString { + c.WriteError(msgWrongType) + return + } + + c.WriteBulk(db.stringGet(opts.key)) + }) +} + +// GETSET +func (m *Miniredis) cmdGetset(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + key, value := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + + old, ok := db.stringKeys[key] + db.stringSet(key, value) + // a GETSET clears the ttl + delete(db.ttl, key) + + if !ok { + c.WriteNull() + return + } + c.WriteBulk(old) + }) +} + +// GETDEL +func (m *Miniredis) cmdGetdel(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + key := args[0] + + if !db.exists(key) { + c.WriteNull() + return + } + + if db.t(key) != keyTypeString { + c.WriteError(msgWrongType) + return + } + + v := db.stringGet(key) + db.del(key, true) + c.WriteBulk(v) + }) +} + +// MGET +func (m *Miniredis) cmdMget(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + c.WriteLen(len(args)) + for _, k := range args { + if t, ok := db.keys[k]; !ok || t != keyTypeString { + c.WriteNull() + continue + } + v, ok := db.stringKeys[k] + if !ok { + // Should not happen, we just checked keys[] + c.WriteNull() + continue + } + c.WriteBulk(v) + } + }) +} + +// INCR +func (m *Miniredis) cmdIncr(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + key := args[0] + if t, ok := db.keys[key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + v, err := db.stringIncr(key, +1) + if err != nil { + c.WriteError(err.Error()) + return + } + // Don't touch TTL + c.WriteInt(v) + }) +} + +// INCRBY +func (m *Miniredis) cmdIncrby(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + var opts struct { + key string + delta int + } + opts.key = args[0] + if ok := optInt(c, args[1], &opts.delta); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + + v, err := db.stringIncr(opts.key, opts.delta) + if err != nil { + c.WriteError(err.Error()) + return + } + // Don't touch TTL + c.WriteInt(v) + }) +} + +// INCRBYFLOAT +func (m *Miniredis) cmdIncrbyfloat(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + key := args[0] + delta, _, err := big.ParseFloat(args[1], 10, 128, 0) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidFloat) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + + v, err := db.stringIncrfloat(key, delta) + if err != nil { + c.WriteError(err.Error()) + return + } + // Don't touch TTL + c.WriteBulk(formatBig(v)) + }) +} + +// DECR +func (m *Miniredis) cmdDecr(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + key := args[0] + if t, ok := db.keys[key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + v, err := db.stringIncr(key, -1) + if err != nil { + c.WriteError(err.Error()) + return + } + // Don't touch TTL + c.WriteInt(v) + }) +} + +// DECRBY +func (m *Miniredis) cmdDecrby(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + var opts struct { + key string + delta int + } + opts.key = args[0] + if ok := optInt(c, args[1], &opts.delta); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + + v, err := db.stringIncr(opts.key, -opts.delta) + if err != nil { + c.WriteError(err.Error()) + return + } + // Don't touch TTL + c.WriteInt(v) + }) +} + +// STRLEN +func (m *Miniredis) cmdStrlen(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(1)) { + return + } + + key := args[0] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + + c.WriteInt(len(db.stringKeys[key])) + }) +} + +// APPEND +func (m *Miniredis) cmdAppend(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + key, value := args[0], args[1] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + + newValue := db.stringKeys[key] + value + db.stringSet(key, newValue) + + c.WriteInt(len(newValue)) + }) +} + +// GETRANGE +func (m *Miniredis) cmdGetrange(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + start int + end int + } + opts.key = args[0] + if ok := optInt(c, args[1], &opts.start); !ok { + return + } + if ok := optInt(c, args[2], &opts.end); !ok { + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + + v := db.stringKeys[opts.key] + c.WriteBulk(withRange(v, opts.start, opts.end)) + }) +} + +// SETRANGE +func (m *Miniredis) cmdSetrange(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + pos int + subst string + } + opts.key = args[0] + if ok := optInt(c, args[1], &opts.pos); !ok { + return + } + if opts.pos < 0 { + setDirty(c) + c.WriteError("ERR offset is out of range") + return + } + opts.subst = args[2] + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + + v := []byte(db.stringKeys[opts.key]) + end := opts.pos + len(opts.subst) + if len(v) < end { + newV := make([]byte, end) + copy(newV, v) + v = newV + } + copy(v[opts.pos:end], opts.subst) + db.stringSet(opts.key, string(v)) + c.WriteInt(len(v)) + }) +} + +// BITCOUNT +func (m *Miniredis) cmdBitcount(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + var opts struct { + useRange bool + start int + end int + key string + } + opts.key, args = args[0], args[1:] + if len(args) >= 2 { + opts.useRange = true + if ok := optInt(c, args[0], &opts.start); !ok { + return + } + if ok := optInt(c, args[1], &opts.end); !ok { + return + } + args = args[2:] + } + if len(args) != 0 { + c.WriteError(msgSyntaxError) + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if !db.exists(opts.key) { + c.WriteInt(0) + return + } + if db.t(opts.key) != keyTypeString { + c.WriteError(msgWrongType) + return + } + + // Real redis only checks after it knows the key is there and a string. + if len(args) != 0 { + c.WriteError(msgSyntaxError) + return + } + + v := db.stringKeys[opts.key] + if opts.useRange { + v = withRange(v, opts.start, opts.end) + } + + c.WriteInt(countBits([]byte(v))) + }) +} + +// BITOP +func (m *Miniredis) cmdBitop(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(3)) { + return + } + + var opts struct { + op string + target string + input []string + } + opts.op = strings.ToUpper(args[0]) + opts.target = args[1] + opts.input = args[2:] + + // 'op' is tested when the transaction is executed. + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + switch opts.op { + case "AND", "OR", "XOR": + first := opts.input[0] + if t, ok := db.keys[first]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + res := []byte(db.stringKeys[first]) + for _, vk := range opts.input[1:] { + if t, ok := db.keys[vk]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + v := db.stringKeys[vk] + cb := map[string]func(byte, byte) byte{ + "AND": func(a, b byte) byte { return a & b }, + "OR": func(a, b byte) byte { return a | b }, + "XOR": func(a, b byte) byte { return a ^ b }, + }[opts.op] + res = sliceBinOp(cb, res, []byte(v)) + } + db.del(opts.target, false) // Keep TTL + if len(res) == 0 { + db.del(opts.target, true) + } else { + db.stringSet(opts.target, string(res)) + } + c.WriteInt(len(res)) + case "NOT": + // NOT only takes a single argument. + if len(opts.input) != 1 { + c.WriteError("ERR BITOP NOT must be called with a single source key.") + return + } + key := opts.input[0] + if t, ok := db.keys[key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + value := []byte(db.stringKeys[key]) + for i := range value { + value[i] = ^value[i] + } + db.del(opts.target, false) // Keep TTL + if len(value) == 0 { + db.del(opts.target, true) + } else { + db.stringSet(opts.target, string(value)) + } + c.WriteInt(len(value)) + default: + c.WriteError(msgSyntaxError) + } + }) +} + +// BITPOS +func (m *Miniredis) cmdBitpos(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, between(2, 4)) { + return + } + + var opts struct { + Key string + Bit int + Start int + End int + WithEnd bool + } + + opts.Key = args[0] + if ok := optInt(c, args[1], &opts.Bit); !ok { + return + } + if len(args) > 2 { + if ok := optInt(c, args[2], &opts.Start); !ok { + return + } + } + if len(args) > 3 { + if ok := optInt(c, args[3], &opts.End); !ok { + return + } + opts.WithEnd = true + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.Key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } else if !ok { + // non-existing key behaves differently + if opts.Bit == 0 { + c.WriteInt(0) + } else { + c.WriteInt(-1) + } + return + } + value := db.stringKeys[opts.Key] + start := opts.Start + end := opts.End + if start < 0 { + start += len(value) + if start < 0 { + start = 0 + } + } + if start > len(value) { + start = len(value) + } + + if opts.WithEnd { + if end < 0 { + end += len(value) + } + if end < 0 { + end = 0 + } + end++ // +1 for redis end semantics + if end > len(value) { + end = len(value) + } + } else { + end = len(value) + } + + if start != 0 || opts.WithEnd { + if end < start { + value = "" + } else { + value = value[start:end] + } + } + pos := bitPos([]byte(value), opts.Bit == 1) + if pos >= 0 { + pos += start * 8 + } + // Special case when looking for 0, but not when start and end are + // given. + if opts.Bit == 0 && pos == -1 && !opts.WithEnd && len(value) > 0 { + pos = start*8 + len(value)*8 + } + c.WriteInt(pos) + }) +} + +// GETBIT +func (m *Miniredis) cmdGetbit(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(2)) { + return + } + + var opts struct { + key string + bit int + } + opts.key = args[0] + if ok := optIntErr(c, args[1], &opts.bit, "ERR bit offset is not an integer or out of range"); !ok { + return + } + if opts.bit < 0 { + setDirty(c) + c.WriteError("ERR bit offset is not an integer or out of range") + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + value := db.stringKeys[opts.key] + + ourByteNr := opts.bit / 8 + var ourByte byte + if ourByteNr > len(value)-1 { + ourByte = '\x00' + } else { + ourByte = value[ourByteNr] + } + res := 0 + if toBits(ourByte)[opts.bit%8] { + res = 1 + } + c.WriteInt(res) + }) +} + +// SETBIT +func (m *Miniredis) cmdSetbit(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(3)) { + return + } + + var opts struct { + key string + bit int + newBit int + } + opts.key = args[0] + if ok := optIntErr(c, args[1], &opts.bit, "ERR bit offset is not an integer or out of range"); !ok { + return + } + if opts.bit < 0 { + setDirty(c) + c.WriteError("ERR bit offset is not an integer or out of range") + return + } + if ok := optIntErr(c, args[2], &opts.newBit, "ERR bit is not an integer or out of range"); !ok { + return + } + if opts.newBit != 0 && opts.newBit != 1 { + setDirty(c) + c.WriteError("ERR bit is not an integer or out of range") + return + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + if t, ok := db.keys[opts.key]; ok && t != keyTypeString { + c.WriteError(msgWrongType) + return + } + value := []byte(db.stringKeys[opts.key]) + + ourByteNr := opts.bit / 8 + ourBitNr := opts.bit % 8 + if ourByteNr > len(value)-1 { + // Too short. Expand. + newValue := make([]byte, ourByteNr+1) + copy(newValue, value) + value = newValue + } + old := 0 + if toBits(value[ourByteNr])[ourBitNr] { + old = 1 + } + if opts.newBit == 0 { + value[ourByteNr] &^= 1 << uint8(7-ourBitNr) + } else { + value[ourByteNr] |= 1 << uint8(7-ourBitNr) + } + db.stringSet(opts.key, string(value)) + + c.WriteInt(old) + }) +} + +// DELEX +func (m *Miniredis) cmdDelex(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, between(1, 4)) { + return + } + + var opts struct { + key string + opEQ bool + matchValue string + } + + // Parse arguments + if len(args) != 1 && len(args) != 3 { + c.WriteError(errWrongNumber("delex")) + return + } + + if len(args) == 3 { + opts.key = args[0] + condition := strings.ToUpper(args[1]) + opts.matchValue = args[2] + + switch condition { + case "IFEQ": + opts.opEQ = true + case "IFNE": + opts.opEQ = false // well doh + case "IFDEQ", "IFDNE": + c.WriteError("ERR unsupported condition for DELEX: " + condition) + return + default: + c.WriteError("Invalid condition. Use IFEQ, IFNE, IFDEQ, or IFDNE") + return + } + } + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + db := m.db(ctx.selectedDB) + + // If no condition is specified, behave like DEL + if opts.key == "" { + key := args[0] + if db.exists(key) { + db.del(key, true) // delete expire + c.WriteInt(1) + } else { + c.WriteInt(0) + } + return + } + + if !db.exists(opts.key) { + c.WriteInt(0) + return + } + + if db.t(opts.key) != keyTypeString { + c.WriteError("ERR Key should be of string type if conditions are specified") + return + } + currentValue := db.stringKeys[opts.key] + + shouldDelete := (currentValue == opts.matchValue) + if !opts.opEQ { + shouldDelete = !shouldDelete + } + if shouldDelete { + db.del(opts.key, true) // delete expire + c.WriteInt(1) + } else { + c.WriteInt(0) + } + }) +} + +// Redis range. both start and end can be negative. +func withRange(v string, start, end int) string { + s, e := redisRange(len(v), start, end, true /* string getrange symantics */) + return v[s:e] +} + +func countBits(v []byte) int { + count := 0 + for _, b := range []byte(v) { + count += bits.OnesCount8(uint8(b)) + } + return count +} + +// sliceBinOp applies an operator to all slice elements, with Redis string +// padding logic. +func sliceBinOp(f func(a, b byte) byte, a, b []byte) []byte { + maxl := len(a) + if len(b) > maxl { + maxl = len(b) + } + lA := make([]byte, maxl) + copy(lA, a) + lB := make([]byte, maxl) + copy(lB, b) + res := make([]byte, maxl) + for i := range res { + res[i] = f(lA[i], lB[i]) + } + return res +} + +// Return the number of the first bit set/unset. +func bitPos(s []byte, bit bool) int { + for i, b := range s { + for j, set := range toBits(b) { + if set == bit { + return i*8 + j + } + } + } + return -1 +} + +// toBits changes a byte in 8 bools. +func toBits(s byte) [8]bool { + r := [8]bool{} + for i := range r { + if s&(uint8(1)< version { + // Abort! Abort! + stopTx(ctx) + c.WriteLen(-1) + return + } + } + + c.WriteLen(len(ctx.transaction)) + for _, cb := range ctx.transaction { + cb(c, ctx) + } + // wake up anyone who waits on anything. + m.signal.Broadcast() + + stopTx(ctx) +} + +// DISCARD +func (m *Miniredis) cmdDiscard(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(0)) { + return + } + + ctx := getCtx(c) + if !inTx(ctx) { + c.WriteError("ERR DISCARD without MULTI") + return + } + + stopTx(ctx) + c.WriteOK() +} + +// WATCH +func (m *Miniredis) cmdWatch(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, atLeast(1)) { + return + } + + ctx := getCtx(c) + if ctx.nested { + c.WriteError(msgNotFromScripts(ctx.nestedSHA)) + return + } + if inTx(ctx) { + c.WriteError("ERR WATCH in MULTI") + return + } + + m.Lock() + defer m.Unlock() + db := m.db(ctx.selectedDB) + + for _, key := range args { + watch(db, ctx, key) + } + c.WriteOK() +} + +// UNWATCH +func (m *Miniredis) cmdUnwatch(c *server.Peer, cmd string, args []string) { + if !m.isValidCMD(c, cmd, args, exactly(0)) { + return + } + + // Doesn't matter if UNWATCH is in a TX or not. Looks like a Redis bug to me. + unwatch(getCtx(c)) + + withTx(m, c, func(c *server.Peer, ctx *connCtx) { + // Do nothing if it's called in a transaction. + c.WriteOK() + }) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/db.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/db.go new file mode 100644 index 00000000..97bdf7c3 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/db.go @@ -0,0 +1,824 @@ +package miniredis + +import ( + "errors" + "fmt" + "math" + "math/big" + "sort" + "strconv" + "time" +) + +var ( + errInvalidEntryID = errors.New("stream ID is invalid") +) + +// exists also updates the lru +func (db *RedisDB) exists(k string) bool { + _, ok := db.keys[k] + if ok { + db.lru[k] = db.master.effectiveNow() + } + return ok +} + +// t gives the type of a key, or "" +func (db *RedisDB) t(k string) string { + return db.keys[k] +} + +// incr increases the version and the lru timestamp +func (db *RedisDB) incr(k string) { + db.lru[k] = db.master.effectiveNow() + db.keyVersion[k]++ +} + +// allKeys returns all keys. Sorted. +func (db *RedisDB) allKeys() []string { + res := make([]string, 0, len(db.keys)) + for k := range db.keys { + res = append(res, k) + } + sort.Strings(res) // To make things deterministic. + return res +} + +// flush removes all keys and values. +func (db *RedisDB) flush() { + db.keys = map[string]string{} + db.lru = map[string]time.Time{} + db.stringKeys = map[string]string{} + db.hashKeys = map[string]hashKey{} + db.listKeys = map[string]listKey{} + db.setKeys = map[string]setKey{} + db.hllKeys = map[string]*hll{} + db.sortedsetKeys = map[string]sortedSet{} + db.ttl = map[string]time.Duration{} + db.hashTTLs = map[string]map[string]time.Duration{} + db.streamKeys = map[string]*streamKey{} +} + +// move something to another db. Will return ok. Or not. +func (db *RedisDB) move(key string, to *RedisDB) bool { + if _, ok := to.keys[key]; ok { + return false + } + + t, ok := db.keys[key] + if !ok { + return false + } + to.keys[key] = db.keys[key] + switch t { + case keyTypeString: + to.stringKeys[key] = db.stringKeys[key] + case keyTypeHash: + to.hashKeys[key] = db.hashKeys[key] + if fieldTTLs, ok := db.hashTTLs[key]; ok { + to.hashTTLs[key] = fieldTTLs + } + case keyTypeList: + to.listKeys[key] = db.listKeys[key] + case keyTypeSet: + to.setKeys[key] = db.setKeys[key] + case keyTypeSortedSet: + to.sortedsetKeys[key] = db.sortedsetKeys[key] + case keyTypeStream: + to.streamKeys[key] = db.streamKeys[key] + case keyTypeHll: + to.hllKeys[key] = db.hllKeys[key] + default: + panic("unhandled key type") + } + if v, ok := db.ttl[key]; ok { + to.ttl[key] = v + } + to.incr(key) + db.del(key, true) + return true +} + +func (db *RedisDB) rename(from, to string) { + db.del(to, true) + switch db.t(from) { + case keyTypeString: + db.stringKeys[to] = db.stringKeys[from] + case keyTypeHash: + db.hashKeys[to] = db.hashKeys[from] + if fieldTTLs, ok := db.hashTTLs[from]; ok { + db.hashTTLs[to] = fieldTTLs + } + case keyTypeList: + db.listKeys[to] = db.listKeys[from] + case keyTypeSet: + db.setKeys[to] = db.setKeys[from] + case keyTypeSortedSet: + db.sortedsetKeys[to] = db.sortedsetKeys[from] + case keyTypeStream: + db.streamKeys[to] = db.streamKeys[from] + case keyTypeHll: + db.hllKeys[to] = db.hllKeys[from] + default: + panic("missing case") + } + db.keys[to] = db.keys[from] + if v, ok := db.ttl[from]; ok { + db.ttl[to] = v + } + db.incr(to) + + db.del(from, true) +} + +func (db *RedisDB) del(k string, delTTL bool) { + if !db.exists(k) { + return + } + t := db.t(k) + delete(db.keys, k) + delete(db.lru, k) + db.keyVersion[k]++ + if delTTL { + delete(db.ttl, k) + } + switch t { + case keyTypeString: + delete(db.stringKeys, k) + case keyTypeHash: + delete(db.hashKeys, k) + delete(db.hashTTLs, k) + case keyTypeList: + delete(db.listKeys, k) + case keyTypeSet: + delete(db.setKeys, k) + case keyTypeSortedSet: + delete(db.sortedsetKeys, k) + case keyTypeStream: + delete(db.streamKeys, k) + case keyTypeHll: + delete(db.hllKeys, k) + default: + panic("Unknown key type: " + t) + } +} + +// stringGet returns the string key or "" on error/nonexists. +func (db *RedisDB) stringGet(k string) string { + if t, ok := db.keys[k]; !ok || t != keyTypeString { + return "" + } + return db.stringKeys[k] +} + +// stringSet force set()s a key. Does not touch expire. +func (db *RedisDB) stringSet(k, v string) { + db.del(k, false) + db.keys[k] = keyTypeString + db.stringKeys[k] = v + db.incr(k) +} + +// change int key value +func (db *RedisDB) stringIncr(k string, delta int) (int, error) { + v := 0 + if sv, ok := db.stringKeys[k]; ok { + var err error + v, err = strconv.Atoi(sv) + if err != nil { + return 0, ErrIntValueError + } + } + + if delta > 0 { + if math.MaxInt-delta < v { + return 0, ErrIntValueOverflowError + } + } else { + if math.MinInt-delta > v { + return 0, ErrIntValueOverflowError + } + } + + v += delta + db.stringSet(k, strconv.Itoa(v)) + return v, nil +} + +// change float key value +func (db *RedisDB) stringIncrfloat(k string, delta *big.Float) (*big.Float, error) { + v := big.NewFloat(0.0) + v.SetPrec(128) + if sv, ok := db.stringKeys[k]; ok { + var err error + v, _, err = big.ParseFloat(sv, 10, 128, 0) + if err != nil { + return nil, ErrFloatValueError + } + } + v.Add(v, delta) + db.stringSet(k, formatBig(v)) + return v, nil +} + +// listLpush is 'left push', aka unshift. Returns the new length. +func (db *RedisDB) listLpush(k, v string) int { + l, ok := db.listKeys[k] + if !ok { + db.keys[k] = keyTypeList + } + l = append([]string{v}, l...) + db.listKeys[k] = l + db.incr(k) + return len(l) +} + +// 'left pop', aka shift. +func (db *RedisDB) listLpop(k string) string { + l := db.listKeys[k] + el := l[0] + l = l[1:] + if len(l) == 0 { + db.del(k, true) + } else { + db.listKeys[k] = l + } + db.incr(k) + return el +} + +func (db *RedisDB) listPush(k string, v ...string) int { + l, ok := db.listKeys[k] + if !ok { + db.keys[k] = keyTypeList + } + l = append(l, v...) + db.listKeys[k] = l + db.incr(k) + return len(l) +} + +func (db *RedisDB) listPop(k string) string { + l := db.listKeys[k] + el := l[len(l)-1] + l = l[:len(l)-1] + if len(l) == 0 { + db.del(k, true) + } else { + db.listKeys[k] = l + db.incr(k) + } + return el +} + +// setset replaces a whole set. +func (db *RedisDB) setSet(k string, set setKey) { + db.keys[k] = keyTypeSet + db.setKeys[k] = set + db.incr(k) +} + +// setadd adds members to a set. Returns nr of new keys. +func (db *RedisDB) setAdd(k string, elems ...string) int { + s, ok := db.setKeys[k] + if !ok { + s = setKey{} + db.keys[k] = keyTypeSet + } + added := 0 + for _, e := range elems { + if _, ok := s[e]; !ok { + added++ + } + s[e] = struct{}{} + } + db.setKeys[k] = s + db.incr(k) + return added +} + +// setrem removes members from a set. Returns nr of deleted keys. +func (db *RedisDB) setRem(k string, fields ...string) int { + s, ok := db.setKeys[k] + if !ok { + return 0 + } + removed := 0 + for _, f := range fields { + if _, ok := s[f]; ok { + removed++ + delete(s, f) + } + } + if len(s) == 0 { + db.del(k, true) + } else { + db.setKeys[k] = s + } + db.incr(k) + return removed +} + +// All members of a set. +func (db *RedisDB) setMembers(k string) []string { + set := db.setKeys[k] + members := make([]string, 0, len(set)) + for k := range set { + members = append(members, k) + } + sort.Strings(members) + return members +} + +// Is a SET value present? +func (db *RedisDB) setIsMember(k, v string) bool { + set, ok := db.setKeys[k] + if !ok { + return false + } + _, ok = set[v] + return ok +} + +// hashFields returns all (sorted) keys ('fields') for a hash key. +func (db *RedisDB) hashFields(k string) []string { + v := db.hashKeys[k] + var r []string + for k := range v { + r = append(r, k) + } + sort.Strings(r) + return r +} + +// hashValues returns all (sorted) values a hash key. +func (db *RedisDB) hashValues(k string) []string { + h := db.hashKeys[k] + var r []string + for _, v := range h { + r = append(r, v) + } + sort.Strings(r) + return r +} + +// hashGet a value +func (db *RedisDB) hashGet(key, field string) string { + return db.hashKeys[key][field] +} + +// hashSet returns the number of new keys +func (db *RedisDB) hashSet(k string, fv ...string) int { + if t, ok := db.keys[k]; ok && t != keyTypeHash { + db.del(k, true) + } + db.keys[k] = keyTypeHash + if _, ok := db.hashKeys[k]; !ok { + db.hashKeys[k] = map[string]string{} + } + new := 0 + for idx := 0; idx < len(fv)-1; idx = idx + 2 { + f, v := fv[idx], fv[idx+1] + _, ok := db.hashKeys[k][f] + db.hashKeys[k][f] = v + db.incr(k) + if !ok { + new++ + } + } + return new +} + +// hashIncr changes int key value +func (db *RedisDB) hashIncr(key, field string, delta int) (int, error) { + v := 0 + if h, ok := db.hashKeys[key]; ok { + if f, ok := h[field]; ok { + var err error + v, err = strconv.Atoi(f) + if err != nil { + return 0, ErrIntValueError + } + } + } + v += delta + db.hashSet(key, field, strconv.Itoa(v)) + return v, nil +} + +// hashIncrfloat changes float key value +func (db *RedisDB) hashIncrfloat(key, field string, delta *big.Float) (*big.Float, error) { + v := big.NewFloat(0.0) + v.SetPrec(128) + if h, ok := db.hashKeys[key]; ok { + if f, ok := h[field]; ok { + var err error + v, _, err = big.ParseFloat(f, 10, 128, 0) + if err != nil { + return nil, ErrFloatValueError + } + } + } + v.Add(v, delta) + db.hashSet(key, field, formatBig(v)) + return v, nil +} + +// sortedSet set returns a sortedSet as map +func (db *RedisDB) sortedSet(key string) map[string]float64 { + ss := db.sortedsetKeys[key] + return map[string]float64(ss) +} + +// ssetSet sets a complete sorted set. +func (db *RedisDB) ssetSet(key string, sset sortedSet) { + db.keys[key] = keyTypeSortedSet + db.incr(key) + db.sortedsetKeys[key] = sset +} + +// ssetAdd adds member to a sorted set. Returns whether this was a new member. +func (db *RedisDB) ssetAdd(key string, score float64, member string) bool { + ss, ok := db.sortedsetKeys[key] + if !ok { + ss = newSortedSet() + db.keys[key] = keyTypeSortedSet + } + _, ok = ss[member] + ss[member] = score + db.sortedsetKeys[key] = ss + db.incr(key) + return !ok +} + +// All members from a sorted set, ordered by score. +func (db *RedisDB) ssetMembers(key string) []string { + ss, ok := db.sortedsetKeys[key] + if !ok { + return nil + } + elems := ss.byScore(asc) + members := make([]string, 0, len(elems)) + for _, e := range elems { + members = append(members, e.member) + } + return members +} + +// All members+scores from a sorted set, ordered by score. +func (db *RedisDB) ssetElements(key string) ssElems { + ss, ok := db.sortedsetKeys[key] + if !ok { + return nil + } + return ss.byScore(asc) +} + +func (db *RedisDB) ssetRandomMember(key string) string { + elems := db.ssetElements(key) + if len(elems) == 0 { + return "" + } + return elems[db.master.randIntn(len(elems))].member +} + +// ssetCard is the sorted set cardinality. +func (db *RedisDB) ssetCard(key string) int { + ss := db.sortedsetKeys[key] + return ss.card() +} + +// ssetRank is the sorted set rank. +func (db *RedisDB) ssetRank(key, member string, d direction) (int, bool) { + ss := db.sortedsetKeys[key] + return ss.rankByScore(member, d) +} + +// ssetScore is sorted set score. +func (db *RedisDB) ssetScore(key, member string) float64 { + ss := db.sortedsetKeys[key] + return ss[member] +} + +// ssetMScore returns multiple scores of a list of members in a sorted set. +func (db *RedisDB) ssetMScore(key string, members []string) []float64 { + scores := make([]float64, 0, len(members)) + ss := db.sortedsetKeys[key] + for _, member := range members { + scores = append(scores, ss[member]) + } + return scores +} + +// ssetRem is sorted set key delete. +func (db *RedisDB) ssetRem(key, member string) bool { + ss := db.sortedsetKeys[key] + _, ok := ss[member] + delete(ss, member) + if len(ss) == 0 { + // Delete key on removal of last member + db.del(key, true) + } + return ok +} + +// ssetExists tells if a member exists in a sorted set. +func (db *RedisDB) ssetExists(key, member string) bool { + ss := db.sortedsetKeys[key] + _, ok := ss[member] + return ok +} + +// ssetIncrby changes float sorted set score. +func (db *RedisDB) ssetIncrby(k, m string, delta float64) float64 { + ss, ok := db.sortedsetKeys[k] + if !ok { + ss = newSortedSet() + db.keys[k] = keyTypeSortedSet + db.sortedsetKeys[k] = ss + } + + v, _ := ss.get(m) + v += delta + ss.set(v, m) + db.incr(k) + return v +} + +// setDiff implements the logic behind SDIFF* +func (db *RedisDB) setDiff(keys []string) (setKey, error) { + key := keys[0] + keys = keys[1:] + if db.exists(key) && db.t(key) != keyTypeSet { + return nil, ErrWrongType + } + s := setKey{} + for k := range db.setKeys[key] { + s[k] = struct{}{} + } + for _, sk := range keys { + if !db.exists(sk) { + continue + } + if db.t(sk) != keyTypeSet { + return nil, ErrWrongType + } + for e := range db.setKeys[sk] { + delete(s, e) + } + } + return s, nil +} + +// setInter implements the logic behind SINTER* +// len keys needs to be > 0 +func (db *RedisDB) setInter(keys []string) (setKey, error) { + // all keys must either not exist, or be of type "set". + for _, key := range keys { + if db.exists(key) && db.t(key) != keyTypeSet { + return nil, ErrWrongType + } + } + + key := keys[0] + keys = keys[1:] + if !db.exists(key) { + return nil, nil + } + if db.t(key) != keyTypeSet { + return nil, ErrWrongType + } + s := setKey{} + for k := range db.setKeys[key] { + s[k] = struct{}{} + } + for _, sk := range keys { + if !db.exists(sk) { + return setKey{}, nil + } + if db.t(sk) != keyTypeSet { + return nil, ErrWrongType + } + other := db.setKeys[sk] + for e := range s { + if _, ok := other[e]; ok { + continue + } + delete(s, e) + } + } + return s, nil +} + +// setIntercard implements the logic behind SINTER* +// len keys needs to be > 0 +func (db *RedisDB) setIntercard(keys []string, limit int) (int, error) { + // all keys must either not exist, or be of type "set". + allExist := true + for _, key := range keys { + exists := db.exists(key) + allExist = allExist && exists + if exists && db.t(key) != "set" { + return 0, ErrWrongType + } + } + + if !allExist { + return 0, nil + } + + smallestKey := keys[0] + smallestIdx := 0 + for i, key := range keys { + if len(db.setKeys[key]) < len(db.setKeys[smallestKey]) { + smallestKey = key + smallestIdx = i + } + } + keys[smallestIdx] = keys[len(keys)-1] + keys = keys[:len(keys)-1] + + count := 0 + for item := range db.setKeys[smallestKey] { + inIntersection := true + for _, key := range keys { + if _, ok := db.setKeys[key][item]; !ok { + inIntersection = false + break + } + } + if inIntersection { + count++ + if count == limit { + break + } + } + } + + return count, nil +} + +// setUnion implements the logic behind SUNION* +func (db *RedisDB) setUnion(keys []string) (setKey, error) { + key := keys[0] + keys = keys[1:] + if db.exists(key) && db.t(key) != "set" { + return nil, ErrWrongType + } + s := setKey{} + for k := range db.setKeys[key] { + s[k] = struct{}{} + } + for _, sk := range keys { + if !db.exists(sk) { + continue + } + if db.t(sk) != "set" { + return nil, ErrWrongType + } + for e := range db.setKeys[sk] { + s[e] = struct{}{} + } + } + return s, nil +} + +func (db *RedisDB) newStream(key string) (*streamKey, error) { + if s, err := db.stream(key); err != nil { + return nil, err + } else if s != nil { + return nil, fmt.Errorf("ErrAlreadyExists") + } + + db.keys[key] = keyTypeStream + s := newStreamKey() + db.streamKeys[key] = s + db.incr(key) + return s, nil +} + +// return existing stream, or nil. +func (db *RedisDB) stream(key string) (*streamKey, error) { + if db.exists(key) && db.t(key) != keyTypeStream { + return nil, ErrWrongType + } + + return db.streamKeys[key], nil +} + +// return existing stream group, or nil. +func (db *RedisDB) streamGroup(key, group string) (*streamGroup, error) { + s, err := db.stream(key) + if err != nil || s == nil { + return nil, err + } + return s.groups[group], nil +} + +// fastForward proceeds the current timestamp with duration, works as a time machine +func (db *RedisDB) fastForward(duration time.Duration) { + for _, key := range db.allKeys() { + if value, ok := db.ttl[key]; ok { + db.ttl[key] = value - duration + db.checkTTL(key) + } + + // Handle hash field TTLs + if db.t(key) == keyTypeHash { + db.checkHashFieldTTL(key, duration) + } + } +} + +func (db *RedisDB) checkHashFieldTTL(key string, duration time.Duration) { + fieldTTLs, ok := db.hashTTLs[key] + if !ok { + return + } + + for field, ttl := range fieldTTLs { + fieldTTLs[field] = ttl - duration + if fieldTTLs[field] <= 0 { + // Delete the expired field + delete(db.hashKeys[key], field) + delete(fieldTTLs, field) + + // If hash is now empty, delete the entire key + if len(db.hashKeys[key]) == 0 { + db.del(key, true) + } + } + } +} + +func (db *RedisDB) checkTTL(key string) { + if v, ok := db.ttl[key]; ok && v <= 0 { + db.del(key, true) + } +} + +// hllAdd adds members to a hll. Returns 1 if at least 1 if internal HyperLogLog was altered, otherwise 0 +func (db *RedisDB) hllAdd(k string, elems ...string) int { + s, ok := db.hllKeys[k] + if !ok { + s = newHll() + db.keys[k] = keyTypeHll + } + hllAltered := 0 + for _, e := range elems { + if s.Add([]byte(e)) { + hllAltered = 1 + } + } + db.hllKeys[k] = s + db.incr(k) + return hllAltered +} + +// hllCount estimates the amount of members added to hll by hllAdd. If called with several arguments, hllCount returns a sum of estimations +func (db *RedisDB) hllCount(keys []string) (int, error) { + countOverall := 0 + for _, key := range keys { + if db.exists(key) && db.t(key) != keyTypeHll { + return 0, ErrNotValidHllValue + } + if !db.exists(key) { + continue + } + countOverall += db.hllKeys[key].Count() + } + + return countOverall, nil +} + +// hllMerge merges all the hlls provided as keys to the first key. Creates a new hll in the first key if it contains nothing +func (db *RedisDB) hllMerge(keys []string) error { + for _, key := range keys { + if db.exists(key) && db.t(key) != keyTypeHll { + return ErrNotValidHllValue + } + } + + destKey := keys[0] + restKeys := keys[1:] + + var destHll *hll + if db.exists(destKey) { + destHll = db.hllKeys[destKey] + } else { + destHll = newHll() + } + + for _, key := range restKeys { + if !db.exists(key) { + continue + } + destHll.Merge(db.hllKeys[key]) + } + + db.hllKeys[destKey] = destHll + db.keys[destKey] = keyTypeHll + db.incr(destKey) + + return nil +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/direct.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/direct.go new file mode 100644 index 00000000..ff527b80 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/direct.go @@ -0,0 +1,862 @@ +package miniredis + +// Commands to modify and query our databases directly. + +import ( + "errors" + "math/big" + "time" +) + +var ( + // ErrKeyNotFound is returned when a key doesn't exist. + ErrKeyNotFound = errors.New(msgKeyNotFound) + + // ErrWrongType when a key is not the right type. + ErrWrongType = errors.New(msgWrongType) + + // ErrNotValidHllValue when a key is not a valid HyperLogLog string value. + ErrNotValidHllValue = errors.New(msgNotValidHllValue) + + // ErrIntValueError can returned by INCRBY + ErrIntValueError = errors.New(msgInvalidInt) + + // ErrIntValueOverflowError can be returned by INCR, DECR, INCRBY, DECRBY + ErrIntValueOverflowError = errors.New(msgIntOverflow) + + // ErrFloatValueError can returned by INCRBYFLOAT + ErrFloatValueError = errors.New(msgInvalidFloat) +) + +// Select sets the DB id for all direct commands. +func (m *Miniredis) Select(i int) { + m.Lock() + defer m.Unlock() + m.selectedDB = i +} + +// Keys returns all keys from the selected database, sorted. +func (m *Miniredis) Keys() []string { + return m.DB(m.selectedDB).Keys() +} + +// Keys returns all keys, sorted. +func (db *RedisDB) Keys() []string { + db.master.Lock() + defer db.master.Unlock() + + return db.allKeys() +} + +// FlushAll removes all keys from all databases. +func (m *Miniredis) FlushAll() { + m.Lock() + defer m.Unlock() + defer m.signal.Broadcast() + + m.flushAll() +} + +func (m *Miniredis) flushAll() { + for _, db := range m.dbs { + db.flush() + } +} + +// FlushDB removes all keys from the selected database. +func (m *Miniredis) FlushDB() { + m.DB(m.selectedDB).FlushDB() +} + +// FlushDB removes all keys. +func (db *RedisDB) FlushDB() { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + db.flush() +} + +// Get returns string keys added with SET. +func (m *Miniredis) Get(k string) (string, error) { + return m.DB(m.selectedDB).Get(k) +} + +// Get returns a string key. +func (db *RedisDB) Get(k string) (string, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(k) { + return "", ErrKeyNotFound + } + if db.t(k) != keyTypeString { + return "", ErrWrongType + } + return db.stringGet(k), nil +} + +// Set sets a string key. Removes expire. +func (m *Miniredis) Set(k, v string) error { + return m.DB(m.selectedDB).Set(k, v) +} + +// Set sets a string key. Removes expire. +// Unlike redis the key can't be an existing non-string key. +func (db *RedisDB) Set(k, v string) error { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if db.exists(k) && db.t(k) != keyTypeString { + return ErrWrongType + } + db.del(k, true) // Remove expire + db.stringSet(k, v) + return nil +} + +// Incr changes a int string value by delta. +func (m *Miniredis) Incr(k string, delta int) (int, error) { + return m.DB(m.selectedDB).Incr(k, delta) +} + +// Incr changes a int string value by delta. +func (db *RedisDB) Incr(k string, delta int) (int, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if db.exists(k) && db.t(k) != keyTypeString { + return 0, ErrWrongType + } + + return db.stringIncr(k, delta) +} + +// IncrByFloat increments the float value of a key by the given delta. +// is an alias for Miniredis.Incrfloat +func (m *Miniredis) IncrByFloat(k string, delta float64) (float64, error) { + return m.Incrfloat(k, delta) +} + +// Incrfloat changes a float string value by delta. +func (m *Miniredis) Incrfloat(k string, delta float64) (float64, error) { + return m.DB(m.selectedDB).Incrfloat(k, delta) +} + +// Incrfloat changes a float string value by delta. +func (db *RedisDB) Incrfloat(k string, delta float64) (float64, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if db.exists(k) && db.t(k) != keyTypeString { + return 0, ErrWrongType + } + + v, err := db.stringIncrfloat(k, big.NewFloat(delta)) + if err != nil { + return 0, err + } + vf, _ := v.Float64() + return vf, nil +} + +// List returns the list k, or an error if it's not there or something else. +// This is the same as the Redis command `LRANGE 0 -1`, but you can do your own +// range-ing. +func (m *Miniredis) List(k string) ([]string, error) { + return m.DB(m.selectedDB).List(k) +} + +// List returns the list k, or an error if it's not there or something else. +// This is the same as the Redis command `LRANGE 0 -1`, but you can do your own +// range-ing. +func (db *RedisDB) List(k string) ([]string, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(k) { + return nil, ErrKeyNotFound + } + if db.t(k) != keyTypeList { + return nil, ErrWrongType + } + return db.listKeys[k], nil +} + +// Lpush prepends one value to a list. Returns the new length. +func (m *Miniredis) Lpush(k, v string) (int, error) { + return m.DB(m.selectedDB).Lpush(k, v) +} + +// Lpush prepends one value to a list. Returns the new length. +func (db *RedisDB) Lpush(k, v string) (int, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if db.exists(k) && db.t(k) != keyTypeList { + return 0, ErrWrongType + } + return db.listLpush(k, v), nil +} + +// Lpop removes and returns the last element in a list. +func (m *Miniredis) Lpop(k string) (string, error) { + return m.DB(m.selectedDB).Lpop(k) +} + +// Lpop removes and returns the last element in a list. +func (db *RedisDB) Lpop(k string) (string, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if !db.exists(k) { + return "", ErrKeyNotFound + } + if db.t(k) != keyTypeList { + return "", ErrWrongType + } + return db.listLpop(k), nil +} + +// RPush appends one or multiple values to a list. Returns the new length. +// An alias for Push +func (m *Miniredis) RPush(k string, v ...string) (int, error) { + return m.Push(k, v...) +} + +// Push add element at the end. Returns the new length. +func (m *Miniredis) Push(k string, v ...string) (int, error) { + return m.DB(m.selectedDB).Push(k, v...) +} + +// Push add element at the end. Is called RPUSH in redis. Returns the new length. +func (db *RedisDB) Push(k string, v ...string) (int, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if db.exists(k) && db.t(k) != keyTypeList { + return 0, ErrWrongType + } + return db.listPush(k, v...), nil +} + +// RPop is an alias for Pop +func (m *Miniredis) RPop(k string) (string, error) { + return m.Pop(k) +} + +// Pop removes and returns the last element. Is called RPOP in Redis. +func (m *Miniredis) Pop(k string) (string, error) { + return m.DB(m.selectedDB).Pop(k) +} + +// Pop removes and returns the last element. Is called RPOP in Redis. +func (db *RedisDB) Pop(k string) (string, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if !db.exists(k) { + return "", ErrKeyNotFound + } + if db.t(k) != keyTypeList { + return "", ErrWrongType + } + + return db.listPop(k), nil +} + +// SAdd adds keys to a set. Returns the number of new keys. +// Alias for SetAdd +func (m *Miniredis) SAdd(k string, elems ...string) (int, error) { + return m.SetAdd(k, elems...) +} + +// SetAdd adds keys to a set. Returns the number of new keys. +func (m *Miniredis) SetAdd(k string, elems ...string) (int, error) { + return m.DB(m.selectedDB).SetAdd(k, elems...) +} + +// SetAdd adds keys to a set. Returns the number of new keys. +func (db *RedisDB) SetAdd(k string, elems ...string) (int, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if db.exists(k) && db.t(k) != keyTypeSet { + return 0, ErrWrongType + } + return db.setAdd(k, elems...), nil +} + +// SMembers returns all keys in a set, sorted. +// Alias for Members. +func (m *Miniredis) SMembers(k string) ([]string, error) { + return m.Members(k) +} + +// Members returns all keys in a set, sorted. +func (m *Miniredis) Members(k string) ([]string, error) { + return m.DB(m.selectedDB).Members(k) +} + +// Members gives all set keys. Sorted. +func (db *RedisDB) Members(k string) ([]string, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(k) { + return nil, ErrKeyNotFound + } + if db.t(k) != keyTypeSet { + return nil, ErrWrongType + } + return db.setMembers(k), nil +} + +// SIsMember tells if value is in the set. +// Alias for IsMember +func (m *Miniredis) SIsMember(k, v string) (bool, error) { + return m.IsMember(k, v) +} + +// IsMember tells if value is in the set. +func (m *Miniredis) IsMember(k, v string) (bool, error) { + return m.DB(m.selectedDB).IsMember(k, v) +} + +// IsMember tells if value is in the set. +func (db *RedisDB) IsMember(k, v string) (bool, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(k) { + return false, ErrKeyNotFound + } + if db.t(k) != keyTypeSet { + return false, ErrWrongType + } + return db.setIsMember(k, v), nil +} + +// HKeys returns all (sorted) keys ('fields') for a hash key. +func (m *Miniredis) HKeys(k string) ([]string, error) { + return m.DB(m.selectedDB).HKeys(k) +} + +// HKeys returns all (sorted) keys ('fields') for a hash key. +func (db *RedisDB) HKeys(key string) ([]string, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(key) { + return nil, ErrKeyNotFound + } + if db.t(key) != keyTypeHash { + return nil, ErrWrongType + } + return db.hashFields(key), nil +} + +// Del deletes a key and any expiration value. Returns whether there was a key. +func (m *Miniredis) Del(k string) bool { + return m.DB(m.selectedDB).Del(k) +} + +// Del deletes a key and any expiration value. Returns whether there was a key. +func (db *RedisDB) Del(k string) bool { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if !db.exists(k) { + return false + } + db.del(k, true) + return true +} + +// Unlink deletes a key and any expiration value. Returns where there was a key. +// It's exactly the same as Del() and is not async. It is here for the consistency. +func (m *Miniredis) Unlink(k string) bool { + return m.Del(k) +} + +// Unlink deletes a key and any expiration value. Returns where there was a key. +// It's exactly the same as Del() and is not async. It is here for the consistency. +func (db *RedisDB) Unlink(k string) bool { + return db.Del(k) +} + +// TTL is the left over time to live. As set via EXPIRE, PEXPIRE, EXPIREAT, +// PEXPIREAT. +// Note: this direct function returns 0 if there is no TTL set, unlike redis, +// which returns -1. +func (m *Miniredis) TTL(k string) time.Duration { + return m.DB(m.selectedDB).TTL(k) +} + +// TTL is the left over time to live. As set via EXPIRE, PEXPIRE, EXPIREAT, +// PEXPIREAT. +// 0 if not set. +func (db *RedisDB) TTL(k string) time.Duration { + db.master.Lock() + defer db.master.Unlock() + + return db.ttl[k] +} + +// SetTTL sets the TTL of a key. +func (m *Miniredis) SetTTL(k string, ttl time.Duration) { + m.DB(m.selectedDB).SetTTL(k, ttl) +} + +// SetTTL sets the time to live of a key. +func (db *RedisDB) SetTTL(k string, ttl time.Duration) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + db.ttl[k] = ttl + db.incr(k) +} + +// Type gives the type of a key, or "" +func (m *Miniredis) Type(k string) string { + return m.DB(m.selectedDB).Type(k) +} + +// Type gives the type of a key, or "" +func (db *RedisDB) Type(k string) string { + db.master.Lock() + defer db.master.Unlock() + + return db.t(k) +} + +// Exists tells whether a key exists. +func (m *Miniredis) Exists(k string) bool { + return m.DB(m.selectedDB).Exists(k) +} + +// Exists tells whether a key exists. +func (db *RedisDB) Exists(k string) bool { + db.master.Lock() + defer db.master.Unlock() + + return db.exists(k) +} + +// HGet returns hash keys added with HSET. +// This will return an empty string if the key is not set. Redis would return +// a nil. +// Returns empty string when the key is of a different type. +func (m *Miniredis) HGet(k, f string) string { + return m.DB(m.selectedDB).HGet(k, f) +} + +// HGet returns hash keys added with HSET. +// Returns empty string when the key is of a different type. +func (db *RedisDB) HGet(k, f string) string { + db.master.Lock() + defer db.master.Unlock() + + h, ok := db.hashKeys[k] + if !ok { + return "" + } + return h[f] +} + +// HSet sets hash keys. +// If there is another key by the same name it will be gone. +func (m *Miniredis) HSet(k string, fv ...string) { + m.DB(m.selectedDB).HSet(k, fv...) +} + +// HSet sets hash keys. +// If there is another key by the same name it will be gone. +func (db *RedisDB) HSet(k string, fv ...string) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + db.hashSet(k, fv...) +} + +// HDel deletes a hash key. +func (m *Miniredis) HDel(k, f string) { + m.DB(m.selectedDB).HDel(k, f) +} + +// HDel deletes a hash key. +func (db *RedisDB) HDel(k, f string) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + db.hdel(k, f) +} + +func (db *RedisDB) hdel(k, f string) { + if _, ok := db.hashKeys[k]; !ok { + return + } + delete(db.hashKeys[k], f) + db.incr(k) +} + +// HIncrBy increases the integer value of a hash field by delta (int). +func (m *Miniredis) HIncrBy(k, f string, delta int) (int, error) { + return m.HIncr(k, f, delta) +} + +// HIncr increases a key/field by delta (int). +func (m *Miniredis) HIncr(k, f string, delta int) (int, error) { + return m.DB(m.selectedDB).HIncr(k, f, delta) +} + +// HIncr increases a key/field by delta (int). +func (db *RedisDB) HIncr(k, f string, delta int) (int, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + return db.hashIncr(k, f, delta) +} + +// HIncrByFloat increases a key/field by delta (float). +func (m *Miniredis) HIncrByFloat(k, f string, delta float64) (float64, error) { + return m.HIncrfloat(k, f, delta) +} + +// HIncrfloat increases a key/field by delta (float). +func (m *Miniredis) HIncrfloat(k, f string, delta float64) (float64, error) { + return m.DB(m.selectedDB).HIncrfloat(k, f, delta) +} + +// HIncrfloat increases a key/field by delta (float). +func (db *RedisDB) HIncrfloat(k, f string, delta float64) (float64, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + v, err := db.hashIncrfloat(k, f, big.NewFloat(delta)) + if err != nil { + return 0, err + } + vf, _ := v.Float64() + return vf, nil +} + +// SRem removes fields from a set. Returns number of deleted fields. +func (m *Miniredis) SRem(k string, fields ...string) (int, error) { + return m.DB(m.selectedDB).SRem(k, fields...) +} + +// SRem removes fields from a set. Returns number of deleted fields. +func (db *RedisDB) SRem(k string, fields ...string) (int, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if !db.exists(k) { + return 0, ErrKeyNotFound + } + if db.t(k) != keyTypeSet { + return 0, ErrWrongType + } + return db.setRem(k, fields...), nil +} + +// ZAdd adds a score,member to a sorted set. +func (m *Miniredis) ZAdd(k string, score float64, member string) (bool, error) { + return m.DB(m.selectedDB).ZAdd(k, score, member) +} + +// ZAdd adds a score,member to a sorted set. +func (db *RedisDB) ZAdd(k string, score float64, member string) (bool, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if db.exists(k) && db.t(k) != keyTypeSortedSet { + return false, ErrWrongType + } + return db.ssetAdd(k, score, member), nil +} + +// ZMembers returns all members of a sorted set by score +func (m *Miniredis) ZMembers(k string) ([]string, error) { + return m.DB(m.selectedDB).ZMembers(k) +} + +// ZMembers returns all members of a sorted set by score +func (db *RedisDB) ZMembers(k string) ([]string, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(k) { + return nil, ErrKeyNotFound + } + if db.t(k) != keyTypeSortedSet { + return nil, ErrWrongType + } + return db.ssetMembers(k), nil +} + +// SortedSet returns a raw string->float64 map. +func (m *Miniredis) SortedSet(k string) (map[string]float64, error) { + return m.DB(m.selectedDB).SortedSet(k) +} + +// SortedSet returns a raw string->float64 map. +func (db *RedisDB) SortedSet(k string) (map[string]float64, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(k) { + return nil, ErrKeyNotFound + } + if db.t(k) != keyTypeSortedSet { + return nil, ErrWrongType + } + return db.sortedSet(k), nil +} + +// ZRem deletes a member. Returns whether the was a key. +func (m *Miniredis) ZRem(k, member string) (bool, error) { + return m.DB(m.selectedDB).ZRem(k, member) +} + +// ZRem deletes a member. Returns whether the was a key. +func (db *RedisDB) ZRem(k, member string) (bool, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + if !db.exists(k) { + return false, ErrKeyNotFound + } + if db.t(k) != keyTypeSortedSet { + return false, ErrWrongType + } + return db.ssetRem(k, member), nil +} + +// ZScore gives the score of a sorted set member. +func (m *Miniredis) ZScore(k, member string) (float64, error) { + return m.DB(m.selectedDB).ZScore(k, member) +} + +// ZScore gives the score of a sorted set member. +func (db *RedisDB) ZScore(k, member string) (float64, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(k) { + return 0, ErrKeyNotFound + } + if db.t(k) != keyTypeSortedSet { + return 0, ErrWrongType + } + return db.ssetScore(k, member), nil +} + +// ZScore gives scores of a list of members in a sorted set. +func (m *Miniredis) ZMScore(k string, members ...string) ([]float64, error) { + return m.DB(m.selectedDB).ZMScore(k, members) +} + +func (db *RedisDB) ZMScore(k string, members []string) ([]float64, error) { + db.master.Lock() + defer db.master.Unlock() + + if !db.exists(k) { + return nil, ErrKeyNotFound + } + if db.t(k) != keyTypeSortedSet { + return nil, ErrWrongType + } + return db.ssetMScore(k, members), nil +} + +// XAdd adds an entry to a stream. `id` can be left empty or be '*'. +// If a value is given normal XADD rules apply. Values should be an even +// length. +func (m *Miniredis) XAdd(k string, id string, values []string) (string, error) { + return m.DB(m.selectedDB).XAdd(k, id, values) +} + +// XAdd adds an entry to a stream. `id` can be left empty or be '*'. +// If a value is given normal XADD rules apply. Values should be an even +// length. +func (db *RedisDB) XAdd(k string, id string, values []string) (string, error) { + db.master.Lock() + defer db.master.Unlock() + defer db.master.signal.Broadcast() + + s, err := db.stream(k) + if err != nil { + return "", err + } + if s == nil { + s, _ = db.newStream(k) + } + + return s.add(id, values, db.master.effectiveNow()) +} + +// Stream returns a slice of stream entries. Oldest first. +func (m *Miniredis) Stream(k string) ([]StreamEntry, error) { + return m.DB(m.selectedDB).Stream(k) +} + +// Stream returns a slice of stream entries. Oldest first. +func (db *RedisDB) Stream(key string) ([]StreamEntry, error) { + db.master.Lock() + defer db.master.Unlock() + + s, err := db.stream(key) + if err != nil { + return nil, err + } + if s == nil { + return nil, nil + } + return s.entries, nil +} + +// Publish a message to subscribers. Returns the number of receivers. +func (m *Miniredis) Publish(channel, message string) int { + m.Lock() + defer m.Unlock() + + return m.publish(channel, message) +} + +// PubSubChannels is "PUBSUB CHANNELS ". An empty pattern is fine +// (meaning all channels). +// Returned channels will be ordered alphabetically. +func (m *Miniredis) PubSubChannels(pattern string) []string { + m.Lock() + defer m.Unlock() + + return activeChannels(m.allSubscribers(), pattern) +} + +// PubSubNumSub is "PUBSUB NUMSUB [channels]". It returns all channels with their +// subscriber count. +func (m *Miniredis) PubSubNumSub(channels ...string) map[string]int { + m.Lock() + defer m.Unlock() + + subs := m.allSubscribers() + res := map[string]int{} + for _, channel := range channels { + res[channel] = countSubs(subs, channel) + } + return res +} + +// PubSubNumPat is "PUBSUB NUMPAT" +func (m *Miniredis) PubSubNumPat() int { + m.Lock() + defer m.Unlock() + + return countPsubs(m.allSubscribers()) +} + +// PfAdd adds keys to a hll. Returns the flag which equals to 1 if the inner hll value has been changed. +func (m *Miniredis) PfAdd(k string, elems ...string) (int, error) { + return m.DB(m.selectedDB).HllAdd(k, elems...) +} + +// HllAdd adds keys to a hll. Returns the flag which equals to true if the inner hll value has been changed. +func (db *RedisDB) HllAdd(k string, elems ...string) (int, error) { + db.master.Lock() + defer db.master.Unlock() + + if db.exists(k) && db.t(k) != keyTypeHll { + return 0, ErrWrongType + } + return db.hllAdd(k, elems...), nil +} + +// PfCount returns an estimation of the amount of elements previously added to a hll. +func (m *Miniredis) PfCount(keys ...string) (int, error) { + return m.DB(m.selectedDB).HllCount(keys...) +} + +// HllCount returns an estimation of the amount of elements previously added to a hll. +func (db *RedisDB) HllCount(keys ...string) (int, error) { + db.master.Lock() + defer db.master.Unlock() + + return db.hllCount(keys) +} + +// PfMerge merges all the input hlls into a hll under destKey key. +func (m *Miniredis) PfMerge(destKey string, sourceKeys ...string) error { + return m.DB(m.selectedDB).HllMerge(destKey, sourceKeys...) +} + +// HllMerge merges all the input hlls into a hll under destKey key. +func (db *RedisDB) HllMerge(destKey string, sourceKeys ...string) error { + db.master.Lock() + defer db.master.Unlock() + + return db.hllMerge(append([]string{destKey}, sourceKeys...)) +} + +// Copy a value. +// Needs the IDs of both the source and dest DBs (which can differ). +// Returns ErrKeyNotFound if src does not exist. +// Overwrites dest if it already exists (unlike the redis command, which needs a flag to allow that). +func (m *Miniredis) Copy(srcDB int, src string, destDB int, dest string) error { + return m.copy(m.DB(srcDB), src, m.DB(destDB), dest) +} + +func (m *Miniredis) SCard(key string) (int, error) { + return m.DB(m.selectedDB).SCard(key) +} + +func (db *RedisDB) SCard(key string) (int, error) { + db.master.Lock() + defer db.master.Unlock() + if !db.exists(key) { + return 0, nil + } + if db.t(key) != "set" { + return 0, ErrWrongType + } + return len(db.setMembers(key)), nil +} + +// return "" if there were no members +func (m *Miniredis) SRandMember(key string) (string, error) { + return m.DB(m.selectedDB).SRandMember(key) +} + +func (db *RedisDB) SRandMember(key string) (string, error) { + db.master.Lock() + defer db.master.Unlock() + if !db.exists(key) { + return "", nil + } + if db.t(key) != "set" { + return "", ErrWrongType + } + members := db.setMembers(key) + if len(members) == 0 { + return "", nil + } + db.master.shuffle(members) + return members[0], nil +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/LICENSE.txt b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/LICENSE.txt new file mode 100644 index 00000000..0a0af2e8 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/LICENSE.txt @@ -0,0 +1,26 @@ +This code is derived from the C code in redis-7.2.0/deps/fpconv/*, which has +this license: + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/Makefile b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/Makefile new file mode 100644 index 00000000..d32d4bdc --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/Makefile @@ -0,0 +1,6 @@ +.PHONY: test fuzz +test: + go test + +fuzz: + go test -fuzz=Fuzz diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/README.md b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/README.md new file mode 100644 index 00000000..c210e603 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/README.md @@ -0,0 +1,3 @@ +This is a translation of the actual C code in Redis (7.2) which does the float +-> string conversion. +Strconv does a close enough job, but we can use the exact same logic, so why not. diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/dtoa.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/dtoa.go new file mode 100644 index 00000000..251fc4f3 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/dtoa.go @@ -0,0 +1,286 @@ +package fpconv + +import ( + "math" +) + +var ( + fracmask uint64 = 0x000FFFFFFFFFFFFF + expmask uint64 = 0x7FF0000000000000 + hiddenbit uint64 = 0x0010000000000000 + signmask uint64 = 0x8000000000000000 + expbias int64 = 1023 + 52 + zeros = []rune("0000000000000000000000") + + tens = []uint64{ + 10000000000000000000, + 1000000000000000000, + 100000000000000000, + 10000000000000000, + 1000000000000000, + 100000000000000, + 10000000000000, + 1000000000000, + 100000000000, + 10000000000, + 1000000000, + 100000000, + 10000000, + 1000000, + 100000, + 10000, + 1000, + 100, + 10, + 1} +) + +func absv(n int) int { + if n < 0 { + return -n + } + return n +} + +func minv(a, b int) int { + if a < b { + return a + } + return b +} + +func Dtoa(d float64) string { + var ( + dest [25]rune // Note C has 24, which is broken + digits [18]rune + + str_len int = 0 + neg = false + ) + + if get_dbits(d)&signmask != 0 { + dest[0] = '-' + str_len++ + neg = true + } + + if spec := filter_special(d, dest[str_len:]); spec != 0 { + return string(dest[:str_len+spec]) + } + + var ( + k int = 0 + ndigits int = grisu2(d, &digits, &k) + ) + + str_len += emit_digits(&digits, ndigits, dest[str_len:], k, neg) + return string(dest[:str_len]) +} + +func filter_special(fp float64, dest []rune) int { + if fp == 0.0 { + dest[0] = '0' + return 1 + } + + if math.IsNaN(fp) { + dest[0] = 'n' + dest[1] = 'a' + dest[2] = 'n' + return 3 + } + if math.IsInf(fp, 0) { + dest[0] = 'i' + dest[1] = 'n' + dest[2] = 'f' + return 3 + } + return 0 +} + +func grisu2(d float64, digits *[18]rune, K *int) int { + w := build_fp(d) + + lower, upper := get_normalized_boundaries(w) + + w = normalize(w) + + var k int64 + cp := find_cachedpow10(upper.exp, &k) + + w = multiply(w, cp) + upper = multiply(upper, cp) + lower = multiply(lower, cp) + + lower.frac++ + upper.frac-- + + *K = int(-k) + + return generate_digits(w, upper, lower, digits[:], K) +} + +func emit_digits(digits *[18]rune, ndigits int, dest []rune, K int, neg bool) int { + exp := int(absv(K + ndigits - 1)) + + /* write plain integer */ + if K >= 0 && (exp < (ndigits + 7)) { + copy(dest, digits[:ndigits]) + copy(dest[ndigits:], zeros[:K]) + + return ndigits + K + } + + /* write decimal w/o scientific notation */ + if K < 0 && (K > -7 || exp < 4) { + offset := int(ndigits - absv(K)) + /* fp < 1.0 -> write leading zero */ + if offset <= 0 { + offset = -offset + dest[0] = '0' + dest[1] = '.' + copy(dest[2:], zeros[:offset]) + copy(dest[offset+2:], digits[:ndigits]) + + return ndigits + 2 + offset + + /* fp > 1.0 */ + } else { + copy(dest, digits[:offset]) + dest[offset] = '.' + copy(dest[offset+1:], digits[offset:offset+ndigits-offset]) + + return ndigits + 1 + } + } + /* write decimal w/ scientific notation */ + l := 18 // was: 18-neg + if neg { + l-- + } + ndigits = minv(ndigits, l) + + var idx int = 0 + dest[idx] = digits[0] + idx++ + + if ndigits > 1 { + dest[idx] = '.' + idx++ + copy(dest[idx:], digits[+1:ndigits-1+1]) + idx += ndigits - 1 + } + + dest[idx] = 'e' + idx++ + + sign := '+' + if K+ndigits-1 < 0 { + sign = '-' + } + dest[idx] = sign + idx++ + + var cent rune = 0 + + if exp > 99 { + cent = rune(exp / 100) + dest[idx] = cent + '0' + idx++ + exp -= int(cent) * 100 + } + if exp > 9 { + dec := rune(exp / 10) + dest[idx] = dec + '0' + idx++ + exp -= int(dec) * 10 + } else if cent != 0 { + dest[idx] = '0' + idx++ + } + + dest[idx] = rune(exp%10) + '0' + idx++ + + return idx +} + +func generate_digits(fp, upper, lower Fp, digits []rune, K *int) int { + var ( + wfrac = uint64(upper.frac - fp.frac) + delta = uint64(upper.frac - lower.frac) + ) + + one := Fp{ + frac: 1 << -upper.exp, + exp: upper.exp, + } + + part1 := uint64(upper.frac >> -one.exp) + part2 := uint64(upper.frac & (one.frac - 1)) + + var ( + idx = 0 + kappa = 10 + index = 10 + ) + /* 1000000000 */ + for ; kappa > 0; index++ { + div := tens[index] + digit := part1 / div + + if digit != 0 || idx != 0 { + digits[idx] = rune(digit) + '0' + idx++ + } + + part1 -= digit * div + kappa-- + + tmp := (part1 << -one.exp) + part2 + if tmp <= delta { + *K += kappa + round_digit(digits, idx, delta, tmp, div<<-one.exp, wfrac) + + return idx + } + } + + /* 10 */ + index = 18 + for { + var unit uint64 = tens[index] + part2 *= 10 + delta *= 10 + kappa-- + + digit := part2 >> -one.exp + if digit != 0 || idx != 0 { + digits[idx] = rune(digit) + '0' + idx++ + } + + part2 &= uint64(one.frac) - 1 + if part2 < delta { + *K += kappa + round_digit(digits, idx, delta, part2, uint64(one.frac), wfrac*unit) + + return idx + } + + index-- + } +} + +func round_digit(digits []rune, + ndigits int, + delta uint64, + rem uint64, + kappa uint64, + frac uint64) { + for rem < frac && delta-rem >= kappa && + (rem+kappa < frac || frac-rem > rem+kappa-frac) { + digits[ndigits-1]-- + rem += kappa + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/fp.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/fp.go new file mode 100644 index 00000000..49064636 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/fp.go @@ -0,0 +1,96 @@ +package fpconv + +import ( + "math" +) + +type ( + Fp struct { + frac uint64 + exp int64 + } +) + +func build_fp(d float64) Fp { + bits := get_dbits(d) + + fp := Fp{ + frac: bits & fracmask, + exp: int64((bits & expmask) >> 52), + } + + if fp.exp != 0 { + fp.frac += hiddenbit + fp.exp -= expbias + } else { + fp.exp = -expbias + 1 + } + + return fp +} + +func normalize(fp Fp) Fp { + for (fp.frac & hiddenbit) == 0 { + fp.frac <<= 1 + fp.exp-- + } + + var shift int64 = 64 - 52 - 1 + fp.frac <<= shift + fp.exp -= shift + return fp +} + +func multiply(a, b Fp) Fp { + lomask := uint64(0x00000000FFFFFFFF) + + var ( + ah_bl = uint64((a.frac >> 32) * (b.frac & lomask)) + al_bh = uint64((a.frac & lomask) * (b.frac >> 32)) + al_bl = uint64((a.frac & lomask) * (b.frac & lomask)) + ah_bh = uint64((a.frac >> 32) * (b.frac >> 32)) + ) + + tmp := uint64((ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32)) + /* round up */ + tmp += uint64(1) << 31 + + return Fp{ + ah_bh + (ah_bl >> 32) + (al_bh >> 32) + (tmp >> 32), + a.exp + b.exp + 64, + } +} + +func get_dbits(d float64) uint64 { + return math.Float64bits(d) +} + +func get_normalized_boundaries(fp Fp) (Fp, Fp) { + upper := Fp{ + frac: (fp.frac << 1) + 1, + exp: fp.exp - 1, + } + for (upper.frac & (hiddenbit << 1)) == 0 { + upper.frac <<= 1 + upper.exp-- + } + + var u_shift int64 = 64 - 52 - 2 + + upper.frac <<= u_shift + upper.exp = upper.exp - u_shift + + l_shift := int64(1) + if fp.frac == hiddenbit { + l_shift = 2 + } + + lower := Fp{ + frac: (fp.frac << l_shift) - 1, + exp: fp.exp - l_shift, + } + + lower.frac <<= lower.exp - upper.exp + lower.exp = upper.exp + return lower, upper +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/powers.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/powers.go new file mode 100644 index 00000000..24725f91 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/fpconv/powers.go @@ -0,0 +1,82 @@ +package fpconv + +var ( + npowers int64 = 87 + steppowers int64 = 8 + firstpower int64 = -348 /* 10 ^ -348 */ + + expmax = -32 + expmin = -60 + + powers_ten = []Fp{ + {18054884314459144840, -1220}, {13451937075301367670, -1193}, + {10022474136428063862, -1166}, {14934650266808366570, -1140}, + {11127181549972568877, -1113}, {16580792590934885855, -1087}, + {12353653155963782858, -1060}, {18408377700990114895, -1034}, + {13715310171984221708, -1007}, {10218702384817765436, -980}, + {15227053142812498563, -954}, {11345038669416679861, -927}, + {16905424996341287883, -901}, {12595523146049147757, -874}, + {9384396036005875287, -847}, {13983839803942852151, -821}, + {10418772551374772303, -794}, {15525180923007089351, -768}, + {11567161174868858868, -741}, {17236413322193710309, -715}, + {12842128665889583758, -688}, {9568131466127621947, -661}, + {14257626930069360058, -635}, {10622759856335341974, -608}, + {15829145694278690180, -582}, {11793632577567316726, -555}, + {17573882009934360870, -529}, {13093562431584567480, -502}, + {9755464219737475723, -475}, {14536774485912137811, -449}, + {10830740992659433045, -422}, {16139061738043178685, -396}, + {12024538023802026127, -369}, {17917957937422433684, -343}, + {13349918974505688015, -316}, {9946464728195732843, -289}, + {14821387422376473014, -263}, {11042794154864902060, -236}, + {16455045573212060422, -210}, {12259964326927110867, -183}, + {18268770466636286478, -157}, {13611294676837538539, -130}, + {10141204801825835212, -103}, {15111572745182864684, -77}, + {11258999068426240000, -50}, {16777216000000000000, -24}, + {12500000000000000000, 3}, {9313225746154785156, 30}, + {13877787807814456755, 56}, {10339757656912845936, 83}, + {15407439555097886824, 109}, {11479437019748901445, 136}, + {17105694144590052135, 162}, {12744735289059618216, 189}, + {9495567745759798747, 216}, {14149498560666738074, 242}, + {10542197943230523224, 269}, {15709099088952724970, 295}, + {11704190886730495818, 322}, {17440603504673385349, 348}, + {12994262207056124023, 375}, {9681479787123295682, 402}, + {14426529090290212157, 428}, {10748601772107342003, 455}, + {16016664761464807395, 481}, {11933345169920330789, 508}, + {17782069995880619868, 534}, {13248674568444952270, 561}, + {9871031767461413346, 588}, {14708983551653345445, 614}, + {10959046745042015199, 641}, {16330252207878254650, 667}, + {12166986024289022870, 694}, {18130221999122236476, 720}, + {13508068024458167312, 747}, {10064294952495520794, 774}, + {14996968138956309548, 800}, {11173611982879273257, 827}, + {16649979327439178909, 853}, {12405201291620119593, 880}, + {9242595204427927429, 907}, {13772540099066387757, 933}, + {10261342003245940623, 960}, {15290591125556738113, 986}, + {11392378155556871081, 1013}, {16975966327722178521, 1039}, + {12648080533535911531, 1066}, + } +) + +func find_cachedpow10(exp int64, k *int64) Fp { + one_log_ten := 0.30102999566398114 + + approx := int64(float64(-(exp + npowers)) * one_log_ten) + idx := int((approx - firstpower) / steppowers) + + for { + current := int(exp + powers_ten[idx].exp + 64) + + if current < expmin { + idx++ + continue + } + + if current > expmax { + idx-- + continue + } + + *k = (firstpower + int64(idx)*steppowers) + + return powers_ten[idx] + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/geo.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geo.go new file mode 100644 index 00000000..3028a167 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geo.go @@ -0,0 +1,46 @@ +package miniredis + +import ( + "math" + + "github.com/alicebob/miniredis/v2/geohash" +) + +func toGeohash(long, lat float64) uint64 { + return geohash.EncodeIntWithPrecision(lat, long, 52) +} + +func fromGeohash(score uint64) (float64, float64) { + lat, long := geohash.DecodeIntWithPrecision(score, 52) + return long, lat +} + +// haversin(θ) function +func hsin(theta float64) float64 { + return math.Pow(math.Sin(theta/2), 2) +} + +// distance function returns the distance (in meters) between two points of +// a given longitude and latitude relatively accurately (using a spherical +// approximation of the Earth) through the Haversin Distance Formula for +// great arc distance on a sphere with accuracy for small distances +// point coordinates are supplied in degrees and converted into rad. in the func +// distance returned is meters +// http://en.wikipedia.org/wiki/Haversine_formula +// Source: https://gist.github.com/cdipaolo/d3f8db3848278b49db68 +func distance(lat1, lon1, lat2, lon2 float64) float64 { + // convert to radians + // must cast radius as float to multiply later + var la1, lo1, la2, lo2 float64 + la1 = lat1 * math.Pi / 180 + lo1 = lon1 * math.Pi / 180 + la2 = lat2 * math.Pi / 180 + lo2 = lon2 * math.Pi / 180 + + earth := 6372797.560856 // Earth radius in METERS, according to src/geohash_helper.c + + // calculate + h := hsin(la2-la1) + math.Cos(la1)*math.Cos(la2)*hsin(lo2-lo1) + + return 2 * earth * math.Asin(math.Sqrt(h)) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/LICENSE b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/LICENSE new file mode 100644 index 00000000..c0190c9a --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Michael McLoughlin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/README.md b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/README.md new file mode 100644 index 00000000..c1a12d14 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/README.md @@ -0,0 +1,2 @@ +This is a (selected) copy of github.com/mmcloughlin/geohash with the latitude +range changed from 90 to ~85, to align with the algorithm use by Redis. diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/base32.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/base32.go new file mode 100644 index 00000000..916b272b --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/base32.go @@ -0,0 +1,44 @@ +package geohash + +// encoding encapsulates an encoding defined by a given base32 alphabet. +type encoding struct { + encode string + decode [256]byte +} + +// newEncoding constructs a new encoding defined by the given alphabet, +// which must be a 32-byte string. +func newEncoding(encoder string) *encoding { + e := new(encoding) + e.encode = encoder + for i := 0; i < len(e.decode); i++ { + e.decode[i] = 0xff + } + for i := 0; i < len(encoder); i++ { + e.decode[encoder[i]] = byte(i) + } + return e +} + +// Decode string into bits of a 64-bit word. The string s may be at most 12 +// characters. +func (e *encoding) Decode(s string) uint64 { + x := uint64(0) + for i := 0; i < len(s); i++ { + x = (x << 5) | uint64(e.decode[s[i]]) + } + return x +} + +// Encode bits of 64-bit word into a string. +func (e *encoding) Encode(x uint64) string { + b := [12]byte{} + for i := 0; i < 12; i++ { + b[11-i] = e.encode[x&0x1f] + x >>= 5 + } + return string(b[:]) +} + +// Base32Encoding with the Geohash alphabet. +var base32encoding = newEncoding("0123456789bcdefghjkmnpqrstuvwxyz") diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/geohash.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/geohash.go new file mode 100644 index 00000000..0e0ca2b2 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/geohash/geohash.go @@ -0,0 +1,269 @@ +// Package geohash provides encoding and decoding of string and integer +// geohashes. +package geohash + +import ( + "math" +) + +const ( + ENC_LAT = 85.05112878 + ENC_LONG = 180.0 +) + +// Direction represents directions in the latitute/longitude space. +type Direction int + +// Cardinal and intercardinal directions +const ( + North Direction = iota + NorthEast + East + SouthEast + South + SouthWest + West + NorthWest +) + +// Encode the point (lat, lng) as a string geohash with the standard 12 +// characters of precision. +func Encode(lat, lng float64) string { + return EncodeWithPrecision(lat, lng, 12) +} + +// EncodeWithPrecision encodes the point (lat, lng) as a string geohash with +// the specified number of characters of precision (max 12). +func EncodeWithPrecision(lat, lng float64, chars uint) string { + bits := 5 * chars + inthash := EncodeIntWithPrecision(lat, lng, bits) + enc := base32encoding.Encode(inthash) + return enc[12-chars:] +} + +// encodeInt provides a Go implementation of integer geohash. This is the +// default implementation of EncodeInt, but optimized versions are provided +// for certain architectures. +func EncodeInt(lat, lng float64) uint64 { + latInt := encodeRange(lat, ENC_LAT) + lngInt := encodeRange(lng, ENC_LONG) + return interleave(latInt, lngInt) +} + +// EncodeIntWithPrecision encodes the point (lat, lng) to an integer with the +// specified number of bits. +func EncodeIntWithPrecision(lat, lng float64, bits uint) uint64 { + hash := EncodeInt(lat, lng) + return hash >> (64 - bits) +} + +// Box represents a rectangle in latitude/longitude space. +type Box struct { + MinLat float64 + MaxLat float64 + MinLng float64 + MaxLng float64 +} + +// Center returns the center of the box. +func (b Box) Center() (lat, lng float64) { + lat = (b.MinLat + b.MaxLat) / 2.0 + lng = (b.MinLng + b.MaxLng) / 2.0 + return +} + +// Contains decides whether (lat, lng) is contained in the box. The +// containment test is inclusive of the edges and corners. +func (b Box) Contains(lat, lng float64) bool { + return (b.MinLat <= lat && lat <= b.MaxLat && + b.MinLng <= lng && lng <= b.MaxLng) +} + +// errorWithPrecision returns the error range in latitude and longitude for in +// integer geohash with bits of precision. +func errorWithPrecision(bits uint) (latErr, lngErr float64) { + b := int(bits) + latBits := b / 2 + lngBits := b - latBits + latErr = math.Ldexp(180.0, -latBits) + lngErr = math.Ldexp(360.0, -lngBits) + return +} + +// BoundingBox returns the region encoded by the given string geohash. +func BoundingBox(hash string) Box { + bits := uint(5 * len(hash)) + inthash := base32encoding.Decode(hash) + return BoundingBoxIntWithPrecision(inthash, bits) +} + +// BoundingBoxIntWithPrecision returns the region encoded by the integer +// geohash with the specified precision. +func BoundingBoxIntWithPrecision(hash uint64, bits uint) Box { + fullHash := hash << (64 - bits) + latInt, lngInt := deinterleave(fullHash) + lat := decodeRange(latInt, ENC_LAT) + lng := decodeRange(lngInt, ENC_LONG) + latErr, lngErr := errorWithPrecision(bits) + return Box{ + MinLat: lat, + MaxLat: lat + latErr, + MinLng: lng, + MaxLng: lng + lngErr, + } +} + +// BoundingBoxInt returns the region encoded by the given 64-bit integer +// geohash. +func BoundingBoxInt(hash uint64) Box { + return BoundingBoxIntWithPrecision(hash, 64) +} + +// DecodeCenter decodes the string geohash to the central point of the bounding box. +func DecodeCenter(hash string) (lat, lng float64) { + box := BoundingBox(hash) + return box.Center() +} + +// DecodeIntWithPrecision decodes the provided integer geohash with bits of +// precision to a (lat, lng) point. +func DecodeIntWithPrecision(hash uint64, bits uint) (lat, lng float64) { + box := BoundingBoxIntWithPrecision(hash, bits) + return box.Center() +} + +// DecodeInt decodes the provided 64-bit integer geohash to a (lat, lng) point. +func DecodeInt(hash uint64) (lat, lng float64) { + return DecodeIntWithPrecision(hash, 64) +} + +// Neighbors returns a slice of geohash strings that correspond to the provided +// geohash's neighbors. +func Neighbors(hash string) []string { + box := BoundingBox(hash) + lat, lng := box.Center() + latDelta := box.MaxLat - box.MinLat + lngDelta := box.MaxLng - box.MinLng + precision := uint(len(hash)) + return []string{ + // N + EncodeWithPrecision(lat+latDelta, lng, precision), + // NE, + EncodeWithPrecision(lat+latDelta, lng+lngDelta, precision), + // E, + EncodeWithPrecision(lat, lng+lngDelta, precision), + // SE, + EncodeWithPrecision(lat-latDelta, lng+lngDelta, precision), + // S, + EncodeWithPrecision(lat-latDelta, lng, precision), + // SW, + EncodeWithPrecision(lat-latDelta, lng-lngDelta, precision), + // W, + EncodeWithPrecision(lat, lng-lngDelta, precision), + // NW + EncodeWithPrecision(lat+latDelta, lng-lngDelta, precision), + } +} + +// NeighborsInt returns a slice of uint64s that correspond to the provided hash's +// neighbors at 64-bit precision. +func NeighborsInt(hash uint64) []uint64 { + return NeighborsIntWithPrecision(hash, 64) +} + +// NeighborsIntWithPrecision returns a slice of uint64s that correspond to the +// provided hash's neighbors at the given precision. +func NeighborsIntWithPrecision(hash uint64, bits uint) []uint64 { + box := BoundingBoxIntWithPrecision(hash, bits) + lat, lng := box.Center() + latDelta := box.MaxLat - box.MinLat + lngDelta := box.MaxLng - box.MinLng + return []uint64{ + // N + EncodeIntWithPrecision(lat+latDelta, lng, bits), + // NE, + EncodeIntWithPrecision(lat+latDelta, lng+lngDelta, bits), + // E, + EncodeIntWithPrecision(lat, lng+lngDelta, bits), + // SE, + EncodeIntWithPrecision(lat-latDelta, lng+lngDelta, bits), + // S, + EncodeIntWithPrecision(lat-latDelta, lng, bits), + // SW, + EncodeIntWithPrecision(lat-latDelta, lng-lngDelta, bits), + // W, + EncodeIntWithPrecision(lat, lng-lngDelta, bits), + // NW + EncodeIntWithPrecision(lat+latDelta, lng-lngDelta, bits), + } +} + +// Neighbor returns a geohash string that corresponds to the provided +// geohash's neighbor in the provided direction +func Neighbor(hash string, direction Direction) string { + return Neighbors(hash)[direction] +} + +// NeighborInt returns a uint64 that corresponds to the provided hash's +// neighbor in the provided direction at 64-bit precision. +func NeighborInt(hash uint64, direction Direction) uint64 { + return NeighborsIntWithPrecision(hash, 64)[direction] +} + +// NeighborIntWithPrecision returns a uint64s that corresponds to the +// provided hash's neighbor in the provided direction at the given precision. +func NeighborIntWithPrecision(hash uint64, bits uint, direction Direction) uint64 { + return NeighborsIntWithPrecision(hash, bits)[direction] +} + +// precalculated for performance +var exp232 = math.Exp2(32) + +// Encode the position of x within the range -r to +r as a 32-bit integer. +func encodeRange(x, r float64) uint32 { + p := (x + r) / (2 * r) + return uint32(p * exp232) +} + +// Decode the 32-bit range encoding X back to a value in the range -r to +r. +func decodeRange(X uint32, r float64) float64 { + p := float64(X) / exp232 + x := 2*r*p - r + return x +} + +// Spread out the 32 bits of x into 64 bits, where the bits of x occupy even +// bit positions. +func spread(x uint32) uint64 { + X := uint64(x) + X = (X | (X << 16)) & 0x0000ffff0000ffff + X = (X | (X << 8)) & 0x00ff00ff00ff00ff + X = (X | (X << 4)) & 0x0f0f0f0f0f0f0f0f + X = (X | (X << 2)) & 0x3333333333333333 + X = (X | (X << 1)) & 0x5555555555555555 + return X +} + +// Interleave the bits of x and y. In the result, x and y occupy even and odd +// bitlevels, respectively. +func interleave(x, y uint32) uint64 { + return spread(x) | (spread(y) << 1) +} + +// Squash the even bitlevels of X into a 32-bit word. Odd bitlevels of X are +// ignored, and may take any value. +func squash(X uint64) uint32 { + X &= 0x5555555555555555 + X = (X | (X >> 1)) & 0x3333333333333333 + X = (X | (X >> 2)) & 0x0f0f0f0f0f0f0f0f + X = (X | (X >> 4)) & 0x00ff00ff00ff00ff + X = (X | (X >> 8)) & 0x0000ffff0000ffff + X = (X | (X >> 16)) & 0x00000000ffffffff + return uint32(X) +} + +// Deinterleave the bits of X into 32-bit words containing the even and odd +// bitlevels of X, respectively. +func deinterleave(X uint64) (uint32, uint32) { + return squash(X), squash(X >> 1) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/LICENSE b/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/LICENSE new file mode 100644 index 00000000..68a49daa --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/README.md b/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/README.md new file mode 100644 index 00000000..0459a1d8 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/README.md @@ -0,0 +1 @@ +Copied from https://github.com/layeh/gopher-json and https://github.com/alicebob/gopher-json diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/json.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/json.go new file mode 100644 index 00000000..11561333 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/gopher-json/json.go @@ -0,0 +1,189 @@ +package json + +import ( + "encoding/json" + "errors" + + "github.com/yuin/gopher-lua" +) + +// Preload adds json to the given Lua state's package.preload table. After it +// has been preloaded, it can be loaded using require: +// +// local json = require("json") +func Preload(L *lua.LState) { + L.PreloadModule("json", Loader) +} + +// Loader is the module loader function. +func Loader(L *lua.LState) int { + t := L.NewTable() + L.SetFuncs(t, api) + L.Push(t) + return 1 +} + +var api = map[string]lua.LGFunction{ + "decode": apiDecode, + "encode": apiEncode, +} + +func apiDecode(L *lua.LState) int { + if L.GetTop() != 1 { + L.Error(lua.LString("bad argument #1 to decode"), 1) + return 0 + } + str := L.CheckString(1) + + value, err := Decode(L, []byte(str)) + if err != nil { + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + L.Push(value) + return 1 +} + +func apiEncode(L *lua.LState) int { + if L.GetTop() != 1 { + L.Error(lua.LString("bad argument #1 to encode"), 1) + return 0 + } + value := L.CheckAny(1) + + data, err := Encode(value) + if err != nil { + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + L.Push(lua.LString(string(data))) + return 1 +} + +var ( + errNested = errors.New("cannot encode recursively nested tables to JSON") + errSparseArray = errors.New("cannot encode sparse array") + errInvalidKeys = errors.New("cannot encode mixed or invalid key types") +) + +type invalidTypeError lua.LValueType + +func (i invalidTypeError) Error() string { + return `cannot encode ` + lua.LValueType(i).String() + ` to JSON` +} + +// Encode returns the JSON encoding of value. +func Encode(value lua.LValue) ([]byte, error) { + return json.Marshal(jsonValue{ + LValue: value, + visited: make(map[*lua.LTable]bool), + }) +} + +type jsonValue struct { + lua.LValue + visited map[*lua.LTable]bool +} + +func (j jsonValue) MarshalJSON() (data []byte, err error) { + switch converted := j.LValue.(type) { + case lua.LBool: + data, err = json.Marshal(bool(converted)) + case lua.LNumber: + data, err = json.Marshal(float64(converted)) + case *lua.LNilType: + data = []byte(`null`) + case lua.LString: + data, err = json.Marshal(string(converted)) + case *lua.LTable: + if j.visited[converted] { + return nil, errNested + } + j.visited[converted] = true + + key, value := converted.Next(lua.LNil) + + switch key.Type() { + case lua.LTNil: // empty table + data = []byte(`[]`) + case lua.LTNumber: + arr := make([]jsonValue, 0, converted.Len()) + expectedKey := lua.LNumber(1) + for key != lua.LNil { + if key.Type() != lua.LTNumber { + err = errInvalidKeys + return + } + if expectedKey != key { + err = errSparseArray + return + } + arr = append(arr, jsonValue{value, j.visited}) + expectedKey++ + key, value = converted.Next(key) + } + data, err = json.Marshal(arr) + case lua.LTString: + obj := make(map[string]jsonValue) + for key != lua.LNil { + if key.Type() != lua.LTString { + err = errInvalidKeys + return + } + obj[key.String()] = jsonValue{value, j.visited} + key, value = converted.Next(key) + } + data, err = json.Marshal(obj) + default: + err = errInvalidKeys + } + default: + err = invalidTypeError(j.LValue.Type()) + } + return +} + +// Decode converts the JSON encoded data to Lua values. +func Decode(L *lua.LState, data []byte) (lua.LValue, error) { + var value interface{} + err := json.Unmarshal(data, &value) + if err != nil { + return nil, err + } + return DecodeValue(L, value), nil +} + +// DecodeValue converts the value to a Lua value. +// +// This function only converts values that the encoding/json package decodes to. +// All other values will return lua.LNil. +func DecodeValue(L *lua.LState, value interface{}) lua.LValue { + switch converted := value.(type) { + case bool: + return lua.LBool(converted) + case float64: + return lua.LNumber(converted) + case string: + return lua.LString(converted) + case json.Number: + return lua.LString(converted) + case []interface{}: + arr := L.CreateTable(len(converted), 0) + for _, item := range converted { + arr.Append(DecodeValue(L, item)) + } + return arr + case map[string]interface{}: + tbl := L.CreateTable(0, len(converted)) + for key, item := range converted { + tbl.RawSetH(lua.LString(key), DecodeValue(L, item)) + } + return tbl + case nil: + return lua.LNil + } + + return lua.LNil +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/hll.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hll.go new file mode 100644 index 00000000..d00ad78a --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hll.go @@ -0,0 +1,42 @@ +package miniredis + +import ( + "github.com/alicebob/miniredis/v2/hyperloglog" +) + +type hll struct { + inner *hyperloglog.Sketch +} + +func newHll() *hll { + return &hll{ + inner: hyperloglog.New14(), + } +} + +// Add returns true if cardinality has been changed, or false otherwise. +func (h *hll) Add(item []byte) bool { + return h.inner.Insert(item) +} + +// Count returns the estimation of a set cardinality. +func (h *hll) Count() int { + return int(h.inner.Estimate()) +} + +// Merge merges the other hll into original one (not making a copy but doing this in place). +func (h *hll) Merge(other *hll) { + _ = h.inner.Merge(other.inner) +} + +// Bytes returns raw-bytes representation of hll data structure. +func (h *hll) Bytes() []byte { + dataBytes, _ := h.inner.MarshalBinary() + return dataBytes +} + +func (h *hll) copy() *hll { + return &hll{ + inner: h.inner.Clone(), + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/LICENSE b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/LICENSE new file mode 100644 index 00000000..8436fdb4 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Axiom Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/README.md b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/README.md new file mode 100644 index 00000000..0fac68df --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/README.md @@ -0,0 +1 @@ +This is a copy of github.com/axiomhq/hyperloglog. \ No newline at end of file diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/compressed.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/compressed.go new file mode 100644 index 00000000..4b908be4 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/compressed.go @@ -0,0 +1,180 @@ +package hyperloglog + +import "encoding/binary" + +// Original author of this file is github.com/clarkduvall/hyperloglog +type iterable interface { + decode(i int, last uint32) (uint32, int) + Len() int + Iter() *iterator +} + +type iterator struct { + i int + last uint32 + v iterable +} + +func (iter *iterator) Next() uint32 { + n, i := iter.v.decode(iter.i, iter.last) + iter.last = n + iter.i = i + return n +} + +func (iter *iterator) Peek() uint32 { + n, _ := iter.v.decode(iter.i, iter.last) + return n +} + +func (iter iterator) HasNext() bool { + return iter.i < iter.v.Len() +} + +type compressedList struct { + count uint32 + last uint32 + b variableLengthList +} + +func (v *compressedList) Clone() *compressedList { + if v == nil { + return nil + } + + newV := &compressedList{ + count: v.count, + last: v.last, + } + + newV.b = make(variableLengthList, len(v.b)) + copy(newV.b, v.b) + return newV +} + +func (v *compressedList) MarshalBinary() (data []byte, err error) { + // Marshal the variableLengthList + bdata, err := v.b.MarshalBinary() + if err != nil { + return nil, err + } + + // At least 4 bytes for the two fixed sized values plus the size of bdata. + data = make([]byte, 0, 4+4+len(bdata)) + + // Marshal the count and last values. + data = append(data, []byte{ + // Number of items in the list. + byte(v.count >> 24), + byte(v.count >> 16), + byte(v.count >> 8), + byte(v.count), + // The last item in the list. + byte(v.last >> 24), + byte(v.last >> 16), + byte(v.last >> 8), + byte(v.last), + }...) + + // Append the list + return append(data, bdata...), nil +} + +func (v *compressedList) UnmarshalBinary(data []byte) error { + if len(data) < 12 { + return ErrorTooShort + } + + // Set the count. + v.count, data = binary.BigEndian.Uint32(data[:4]), data[4:] + + // Set the last value. + v.last, data = binary.BigEndian.Uint32(data[:4]), data[4:] + + // Set the list. + sz, data := binary.BigEndian.Uint32(data[:4]), data[4:] + v.b = make([]uint8, sz) + if uint32(len(data)) < sz { + return ErrorTooShort + } + for i := uint32(0); i < sz; i++ { + v.b[i] = data[i] + } + return nil +} + +func newCompressedList() *compressedList { + v := &compressedList{} + v.b = make(variableLengthList, 0) + return v +} + +func (v *compressedList) Len() int { + return len(v.b) +} + +func (v *compressedList) decode(i int, last uint32) (uint32, int) { + n, i := v.b.decode(i, last) + return n + last, i +} + +func (v *compressedList) Append(x uint32) { + v.count++ + v.b = v.b.Append(x - v.last) + v.last = x +} + +func (v *compressedList) Iter() *iterator { + return &iterator{0, 0, v} +} + +type variableLengthList []uint8 + +func (v variableLengthList) MarshalBinary() (data []byte, err error) { + // 4 bytes for the size of the list, and a byte for each element in the + // list. + data = make([]byte, 0, 4+v.Len()) + + // Length of the list. We only need 32 bits because the size of the set + // couldn't exceed that on 32 bit architectures. + sz := v.Len() + data = append(data, []byte{ + byte(sz >> 24), + byte(sz >> 16), + byte(sz >> 8), + byte(sz), + }...) + + // Marshal each element in the list. + for i := 0; i < sz; i++ { + data = append(data, v[i]) + } + + return data, nil +} + +func (v variableLengthList) Len() int { + return len(v) +} + +func (v *variableLengthList) Iter() *iterator { + return &iterator{0, 0, v} +} + +func (v variableLengthList) decode(i int, last uint32) (uint32, int) { + var x uint32 + j := i + for ; v[j]&0x80 != 0; j++ { + x |= uint32(v[j]&0x7f) << (uint(j-i) * 7) + } + x |= uint32(v[j]) << (uint(j-i) * 7) + return x, j + 1 +} + +func (v variableLengthList) Append(x uint32) variableLengthList { + for x&0xffffff80 != 0 { + v = append(v, uint8((x&0x7f)|0x80)) + x >>= 7 + } + return append(v, uint8(x&0x7f)) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/hyperloglog.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/hyperloglog.go new file mode 100644 index 00000000..82663915 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/hyperloglog.go @@ -0,0 +1,424 @@ +package hyperloglog + +import ( + "encoding/binary" + "errors" + "fmt" + "math" + "sort" +) + +const ( + capacity = uint8(16) + pp = uint8(25) + mp = uint32(1) << pp + version = 1 +) + +// Sketch is a HyperLogLog data-structure for the count-distinct problem, +// approximating the number of distinct elements in a multiset. +type Sketch struct { + p uint8 + b uint8 + m uint32 + alpha float64 + tmpSet set + sparseList *compressedList + regs *registers +} + +// New returns a HyperLogLog Sketch with 2^14 registers (precision 14) +func New() *Sketch { + return New14() +} + +// New14 returns a HyperLogLog Sketch with 2^14 registers (precision 14) +func New14() *Sketch { + sk, _ := newSketch(14, true) + return sk +} + +// New16 returns a HyperLogLog Sketch with 2^16 registers (precision 16) +func New16() *Sketch { + sk, _ := newSketch(16, true) + return sk +} + +// NewNoSparse returns a HyperLogLog Sketch with 2^14 registers (precision 14) +// that will not use a sparse representation +func NewNoSparse() *Sketch { + sk, _ := newSketch(14, false) + return sk +} + +// New16NoSparse returns a HyperLogLog Sketch with 2^16 registers (precision 16) +// that will not use a sparse representation +func New16NoSparse() *Sketch { + sk, _ := newSketch(16, false) + return sk +} + +// newSketch returns a HyperLogLog Sketch with 2^precision registers +func newSketch(precision uint8, sparse bool) (*Sketch, error) { + if precision < 4 || precision > 18 { + return nil, fmt.Errorf("p has to be >= 4 and <= 18") + } + m := uint32(math.Pow(2, float64(precision))) + s := &Sketch{ + m: m, + p: precision, + alpha: alpha(float64(m)), + } + if sparse { + s.tmpSet = set{} + s.sparseList = newCompressedList() + } else { + s.regs = newRegisters(m) + } + return s, nil +} + +func (sk *Sketch) sparse() bool { + return sk.sparseList != nil +} + +// Clone returns a deep copy of sk. +func (sk *Sketch) Clone() *Sketch { + return &Sketch{ + b: sk.b, + p: sk.p, + m: sk.m, + alpha: sk.alpha, + tmpSet: sk.tmpSet.Clone(), + sparseList: sk.sparseList.Clone(), + regs: sk.regs.clone(), + } +} + +// Converts to normal if the sparse list is too large. +func (sk *Sketch) maybeToNormal() { + if uint32(len(sk.tmpSet))*100 > sk.m { + sk.mergeSparse() + if uint32(sk.sparseList.Len()) > sk.m { + sk.toNormal() + } + } +} + +// Merge takes another Sketch and combines it with Sketch h. +// If Sketch h is using the sparse Sketch, it will be converted +// to the normal Sketch. +func (sk *Sketch) Merge(other *Sketch) error { + if other == nil { + // Nothing to do + return nil + } + cpOther := other.Clone() + + if sk.p != cpOther.p { + return errors.New("precisions must be equal") + } + + if sk.sparse() && other.sparse() { + for k := range other.tmpSet { + sk.tmpSet.add(k) + } + for iter := other.sparseList.Iter(); iter.HasNext(); { + sk.tmpSet.add(iter.Next()) + } + sk.maybeToNormal() + return nil + } + + if sk.sparse() { + sk.toNormal() + } + + if cpOther.sparse() { + for k := range cpOther.tmpSet { + i, r := decodeHash(k, cpOther.p, pp) + sk.insert(i, r) + } + + for iter := cpOther.sparseList.Iter(); iter.HasNext(); { + i, r := decodeHash(iter.Next(), cpOther.p, pp) + sk.insert(i, r) + } + } else { + if sk.b < cpOther.b { + sk.regs.rebase(cpOther.b - sk.b) + sk.b = cpOther.b + } else { + cpOther.regs.rebase(sk.b - cpOther.b) + cpOther.b = sk.b + } + + for i, v := range cpOther.regs.tailcuts { + v1 := v.get(0) + if v1 > sk.regs.get(uint32(i)*2) { + sk.regs.set(uint32(i)*2, v1) + } + v2 := v.get(1) + if v2 > sk.regs.get(1+uint32(i)*2) { + sk.regs.set(1+uint32(i)*2, v2) + } + } + } + return nil +} + +// Convert from sparse Sketch to dense Sketch. +func (sk *Sketch) toNormal() { + if len(sk.tmpSet) > 0 { + sk.mergeSparse() + } + + sk.regs = newRegisters(sk.m) + for iter := sk.sparseList.Iter(); iter.HasNext(); { + i, r := decodeHash(iter.Next(), sk.p, pp) + sk.insert(i, r) + } + + sk.tmpSet = nil + sk.sparseList = nil +} + +func (sk *Sketch) insert(i uint32, r uint8) bool { + changed := false + if r-sk.b >= capacity { + //overflow + db := sk.regs.min() + if db > 0 { + sk.b += db + sk.regs.rebase(db) + changed = true + } + } + if r > sk.b { + val := r - sk.b + if c1 := capacity - 1; c1 < val { + val = c1 + } + + if val > sk.regs.get(i) { + sk.regs.set(i, val) + changed = true + } + } + return changed +} + +// Insert adds element e to sketch +func (sk *Sketch) Insert(e []byte) bool { + x := hash(e) + return sk.InsertHash(x) +} + +// InsertHash adds hash x to sketch +func (sk *Sketch) InsertHash(x uint64) bool { + if sk.sparse() { + changed := sk.tmpSet.add(encodeHash(x, sk.p, pp)) + if !changed { + return false + } + if uint32(len(sk.tmpSet))*100 > sk.m/2 { + sk.mergeSparse() + if uint32(sk.sparseList.Len()) > sk.m/2 { + sk.toNormal() + } + } + return true + } else { + i, r := getPosVal(x, sk.p) + return sk.insert(uint32(i), r) + } +} + +// Estimate returns the cardinality of the Sketch +func (sk *Sketch) Estimate() uint64 { + if sk.sparse() { + sk.mergeSparse() + return uint64(linearCount(mp, mp-sk.sparseList.count)) + } + + sum, ez := sk.regs.sumAndZeros(sk.b) + m := float64(sk.m) + var est float64 + + var beta func(float64) float64 + if sk.p < 16 { + beta = beta14 + } else { + beta = beta16 + } + + if sk.b == 0 { + est = (sk.alpha * m * (m - ez) / (sum + beta(ez))) + } else { + est = (sk.alpha * m * m / sum) + } + + return uint64(est + 0.5) +} + +func (sk *Sketch) mergeSparse() { + if len(sk.tmpSet) == 0 { + return + } + + keys := make(uint64Slice, 0, len(sk.tmpSet)) + for k := range sk.tmpSet { + keys = append(keys, k) + } + sort.Sort(keys) + + newList := newCompressedList() + for iter, i := sk.sparseList.Iter(), 0; iter.HasNext() || i < len(keys); { + if !iter.HasNext() { + newList.Append(keys[i]) + i++ + continue + } + + if i >= len(keys) { + newList.Append(iter.Next()) + continue + } + + x1, x2 := iter.Peek(), keys[i] + if x1 == x2 { + newList.Append(iter.Next()) + i++ + } else if x1 > x2 { + newList.Append(x2) + i++ + } else { + newList.Append(iter.Next()) + } + } + + sk.sparseList = newList + sk.tmpSet = set{} +} + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +func (sk *Sketch) MarshalBinary() (data []byte, err error) { + // Marshal a version marker. + data = append(data, version) + // Marshal p. + data = append(data, sk.p) + // Marshal b + data = append(data, sk.b) + + if sk.sparse() { + // It's using the sparse Sketch. + data = append(data, byte(1)) + + // Add the tmp_set + tsdata, err := sk.tmpSet.MarshalBinary() + if err != nil { + return nil, err + } + data = append(data, tsdata...) + + // Add the sparse Sketch + sdata, err := sk.sparseList.MarshalBinary() + if err != nil { + return nil, err + } + return append(data, sdata...), nil + } + + // It's using the dense Sketch. + data = append(data, byte(0)) + + // Add the dense sketch Sketch. + sz := len(sk.regs.tailcuts) + data = append(data, []byte{ + byte(sz >> 24), + byte(sz >> 16), + byte(sz >> 8), + byte(sz), + }...) + + // Marshal each element in the list. + for i := 0; i < len(sk.regs.tailcuts); i++ { + data = append(data, byte(sk.regs.tailcuts[i])) + } + + return data, nil +} + +// ErrorTooShort is an error that UnmarshalBinary try to parse too short +// binary. +var ErrorTooShort = errors.New("too short binary") + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +func (sk *Sketch) UnmarshalBinary(data []byte) error { + if len(data) < 8 { + return ErrorTooShort + } + + // Unmarshal version. We may need this in the future if we make + // non-compatible changes. + _ = data[0] + + // Unmarshal p. + p := data[1] + + // Unmarshal b. + sk.b = data[2] + + // Determine if we need a sparse Sketch + sparse := data[3] == byte(1) + + // Make a newSketch Sketch if the precision doesn't match or if the Sketch was used + if sk.p != p || sk.regs != nil || len(sk.tmpSet) > 0 || (sk.sparseList != nil && sk.sparseList.Len() > 0) { + newh, err := newSketch(p, sparse) + if err != nil { + return err + } + newh.b = sk.b + *sk = *newh + } + + // h is now initialised with the correct p. We just need to fill the + // rest of the details out. + if sparse { + // Using the sparse Sketch. + + // Unmarshal the tmp_set. + tssz := binary.BigEndian.Uint32(data[4:8]) + sk.tmpSet = make(map[uint32]struct{}, tssz) + + // We need to unmarshal tssz values in total, and each value requires us + // to read 4 bytes. + tsLastByte := int((tssz * 4) + 8) + for i := 8; i < tsLastByte; i += 4 { + k := binary.BigEndian.Uint32(data[i : i+4]) + sk.tmpSet[k] = struct{}{} + } + + // Unmarshal the sparse Sketch. + return sk.sparseList.UnmarshalBinary(data[tsLastByte:]) + } + + // Using the dense Sketch. + sk.sparseList = nil + sk.tmpSet = nil + dsz := binary.BigEndian.Uint32(data[4:8]) + sk.regs = newRegisters(dsz * 2) + data = data[8:] + + for i, val := range data { + sk.regs.tailcuts[i] = reg(val) + if uint8(sk.regs.tailcuts[i]<<4>>4) > 0 { + sk.regs.nz-- + } + if uint8(sk.regs.tailcuts[i]>>4) > 0 { + sk.regs.nz-- + } + } + + return nil +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/registers.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/registers.go new file mode 100644 index 00000000..19bb5d47 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/registers.go @@ -0,0 +1,114 @@ +package hyperloglog + +import ( + "math" +) + +type reg uint8 +type tailcuts []reg + +type registers struct { + tailcuts + nz uint32 +} + +func (r *reg) set(offset, val uint8) bool { + var isZero bool + if offset == 0 { + isZero = *r < 16 + tmpVal := uint8((*r) << 4 >> 4) + *r = reg(tmpVal | (val << 4)) + } else { + isZero = *r&0x0f == 0 + tmpVal := uint8((*r) >> 4 << 4) + *r = reg(tmpVal | val) + } + return isZero +} + +func (r *reg) get(offset uint8) uint8 { + if offset == 0 { + return uint8((*r) >> 4) + } + return uint8((*r) << 4 >> 4) +} + +func newRegisters(size uint32) *registers { + return ®isters{ + tailcuts: make(tailcuts, size/2), + nz: size, + } +} + +func (rs *registers) clone() *registers { + if rs == nil { + return nil + } + tc := make([]reg, len(rs.tailcuts)) + copy(tc, rs.tailcuts) + return ®isters{ + tailcuts: tc, + nz: rs.nz, + } +} + +func (rs *registers) rebase(delta uint8) { + nz := uint32(len(rs.tailcuts)) * 2 + for i := range rs.tailcuts { + for j := uint8(0); j < 2; j++ { + val := rs.tailcuts[i].get(j) + if val >= delta { + rs.tailcuts[i].set(j, val-delta) + if val-delta > 0 { + nz-- + } + } + } + } + rs.nz = nz +} + +func (rs *registers) set(i uint32, val uint8) { + offset, index := uint8(i)&1, i/2 + if rs.tailcuts[index].set(offset, val) { + rs.nz-- + } +} + +func (rs *registers) get(i uint32) uint8 { + offset, index := uint8(i)&1, i/2 + return rs.tailcuts[index].get(offset) +} + +func (rs *registers) sumAndZeros(base uint8) (res, ez float64) { + for _, r := range rs.tailcuts { + for j := uint8(0); j < 2; j++ { + v := float64(base + r.get(j)) + if v == 0 { + ez++ + } + res += 1.0 / math.Pow(2.0, v) + } + } + rs.nz = uint32(ez) + return res, ez +} + +func (rs *registers) min() uint8 { + if rs.nz > 0 { + return 0 + } + min := uint8(math.MaxUint8) + for _, r := range rs.tailcuts { + if r == 0 || min == 0 { + return 0 + } + if val := uint8(r << 4 >> 4); val < min { + min = val + } + if val := uint8(r >> 4); val < min { + min = val + } + } + return min +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/sparse.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/sparse.go new file mode 100644 index 00000000..8c457d32 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/sparse.go @@ -0,0 +1,92 @@ +package hyperloglog + +import ( + "math/bits" +) + +func getIndex(k uint32, p, pp uint8) uint32 { + if k&1 == 1 { + return bextr32(k, 32-p, p) + } + return bextr32(k, pp-p+1, p) +} + +// Encode a hash to be used in the sparse representation. +func encodeHash(x uint64, p, pp uint8) uint32 { + idx := uint32(bextr(x, 64-pp, pp)) + if bextr(x, 64-pp, pp-p) == 0 { + zeros := bits.LeadingZeros64((bextr(x, 0, 64-pp)<> 24), + byte(sl >> 16), + byte(sl >> 8), + byte(sl), + }...) + + // Marshal each element in the set. + for k := range s { + data = append(data, []byte{ + byte(k >> 24), + byte(k >> 16), + byte(k >> 8), + byte(k), + }...) + } + + return data, nil +} + +type uint64Slice []uint32 + +func (p uint64Slice) Len() int { return len(p) } +func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/utils.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/utils.go new file mode 100644 index 00000000..896bf7e7 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/hyperloglog/utils.go @@ -0,0 +1,69 @@ +package hyperloglog + +import ( + "github.com/alicebob/miniredis/v2/metro" + "math" + "math/bits" +) + +var hash = hashFunc + +func beta14(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.370393911*ez + + 0.070471823*zl + + 0.17393686*math.Pow(zl, 2) + + 0.16339839*math.Pow(zl, 3) + + -0.09237745*math.Pow(zl, 4) + + 0.03738027*math.Pow(zl, 5) + + -0.005384159*math.Pow(zl, 6) + + 0.00042419*math.Pow(zl, 7) +} + +func beta16(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.37331876643753059*ez + + -1.41704077448122989*zl + + 0.40729184796612533*math.Pow(zl, 2) + + 1.56152033906584164*math.Pow(zl, 3) + + -0.99242233534286128*math.Pow(zl, 4) + + 0.26064681399483092*math.Pow(zl, 5) + + -0.03053811369682807*math.Pow(zl, 6) + + 0.00155770210179105*math.Pow(zl, 7) +} + +func alpha(m float64) float64 { + switch m { + case 16: + return 0.673 + case 32: + return 0.697 + case 64: + return 0.709 + } + return 0.7213 / (1 + 1.079/m) +} + +func getPosVal(x uint64, p uint8) (uint64, uint8) { + i := bextr(x, 64-p, p) // {x63,...,x64-p} + w := x<

> start) & ((1 << length) - 1) +} + +func bextr32(v uint32, start, length uint8) uint32 { + return (v >> start) & ((1 << length) - 1) +} + +func hashFunc(e []byte) uint64 { + return metro.Hash64(e, 1337) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/keys.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/keys.go new file mode 100644 index 00000000..058e0a79 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/keys.go @@ -0,0 +1,83 @@ +package miniredis + +// Translate the 'KEYS' or 'PSUBSCRIBE' argument ('foo*', 'f??', &c.) into a regexp. + +import ( + "bytes" + "regexp" +) + +// patternRE compiles a glob to a regexp. Returns nil if the given +// pattern will never match anything. +// The general strategy is to sandwich all non-meta characters between \Q...\E. +func patternRE(k string) *regexp.Regexp { + re := bytes.Buffer{} + re.WriteString(`(?s)^\Q`) + for i := 0; i < len(k); i++ { + p := k[i] + switch p { + case '*': + re.WriteString(`\E.*\Q`) + case '?': + re.WriteString(`\E.\Q`) + case '[': + charClass := bytes.Buffer{} + i++ + for ; i < len(k); i++ { + if k[i] == ']' { + break + } + if k[i] == '\\' { + if i == len(k)-1 { + // Ends with a '\'. U-huh. + return nil + } + charClass.WriteByte(k[i]) + i++ + charClass.WriteByte(k[i]) + continue + } + charClass.WriteByte(k[i]) + } + if charClass.Len() == 0 { + // '[]' is valid in Redis, but matches nothing. + return nil + } + re.WriteString(`\E[`) + re.Write(charClass.Bytes()) + re.WriteString(`]\Q`) + + case '\\': + if i == len(k)-1 { + // Ends with a '\'. U-huh. + return nil + } + // Forget the \, keep the next char. + i++ + re.WriteByte(k[i]) + continue + default: + re.WriteByte(p) + } + } + re.WriteString(`\E$`) + return regexp.MustCompile(re.String()) +} + +// matchKeys filters only matching keys. +// The returned boolean is whether the match pattern was valid +func matchKeys(keys []string, match string) ([]string, bool) { + re := patternRE(match) + if re == nil { + // Special case: the given pattern won't match anything or is invalid. + return nil, false + } + var res []string + for _, k := range keys { + if !re.MatchString(k) { + continue + } + res = append(res, k) + } + return res, true +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/lua.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/lua.go new file mode 100644 index 00000000..29f3aeae --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/lua.go @@ -0,0 +1,312 @@ +package miniredis + +import ( + "bufio" + "bytes" + "fmt" + "strings" + + lua "github.com/yuin/gopher-lua" + + "github.com/alicebob/miniredis/v2/server" +) + +var luaRedisConstants = map[string]lua.LValue{ + "LOG_DEBUG": lua.LNumber(0), + "LOG_VERBOSE": lua.LNumber(1), + "LOG_NOTICE": lua.LNumber(2), + "LOG_WARNING": lua.LNumber(3), +} + +func mkLua(srv *server.Server, c *server.Peer, sha string, readOnly bool) (map[string]lua.LGFunction, map[string]lua.LValue) { + mkCall := func(failFast bool) func(l *lua.LState) int { + // one server.Ctx for a single Lua run + pCtx := &connCtx{} + if getCtx(c).authenticated { + pCtx.authenticated = true + } + pCtx.nested = true + pCtx.nestedSHA = sha + pCtx.selectedDB = getCtx(c).selectedDB + + return func(l *lua.LState) int { + top := l.GetTop() + if top == 0 { + l.Error(lua.LString(fmt.Sprintf("Please specify at least one argument for this redis lib call script: %s, &c.", sha)), 1) + return 0 + } + var args []string + for i := 1; i <= top; i++ { + switch a := l.Get(i).(type) { + case lua.LNumber: + args = append(args, a.String()) + case lua.LString: + args = append(args, string(a)) + default: + l.Error(lua.LString(fmt.Sprintf("Lua redis lib command arguments must be strings or integers script: %s, &c.", sha)), 1) + return 0 + } + } + if len(args) == 0 { + l.Error(lua.LString(msgNotFromScripts(sha)), 1) + return 0 + } + + if readOnly && len(args) > 0 { + if srv.IsRegisteredCommand(args[0]) && !srv.IsReadOnlyCommand(args[0]) { + if failFast { + l.Error(lua.LString("Write commands are not allowed in read-only scripts"), 1) + return 0 + } + // pcall() mode - return error table + res := &lua.LTable{} + res.RawSetString("err", lua.LString("Write commands are not allowed in read-only scripts")) + l.Push(res) + return 1 + } + } + + buf := &bytes.Buffer{} + wr := bufio.NewWriter(buf) + peer := server.NewPeer(wr) + peer.Ctx = pCtx + srv.Dispatch(peer, args) + wr.Flush() + + res, err := server.ParseReply(bufio.NewReader(buf)) + if err != nil { + if failFast { + // call() mode + if strings.Contains(err.Error(), "ERR unknown command") { + l.Error(lua.LString(fmt.Sprintf("Unknown Redis command called from script script: %s, &c.", sha)), 1) + } else { + l.Error(lua.LString(err.Error()), 1) + } + return 0 + } + // pcall() mode + res := &lua.LTable{} + if strings.Contains(err.Error(), "ERR unknown command") { + res.RawSetString("err", lua.LString("ERR Unknown Redis command called from script")) + } else { + res.RawSetString("err", lua.LString(err.Error())) + } + l.Push(res) + return 1 + } + + if res == nil { + l.Push(lua.LFalse) + } else { + switch r := res.(type) { + case int64: + l.Push(lua.LNumber(r)) + case int: + l.Push(lua.LNumber(r)) + case []uint8: + l.Push(lua.LString(string(r))) + case []interface{}: + l.Push(redisToLua(l, r)) + case server.Simple: + l.Push(luaStatusReply(string(r))) + case string: + l.Push(lua.LString(r)) + case error: + l.Error(lua.LString(r.Error()), 1) + return 0 + default: + panic(fmt.Sprintf("type not handled (%T)", r)) + } + } + return 1 + } + } + + return map[string]lua.LGFunction{ + "call": mkCall(true), + "pcall": mkCall(false), + "error_reply": func(l *lua.LState) int { + v := l.Get(1) + msg, ok := v.(lua.LString) + if !ok { + l.Error(lua.LString("wrong number or type of arguments"), 1) + return 0 + } + res := &lua.LTable{} + parts := strings.SplitN(msg.String(), " ", 2) + // '-' at the beginging will be added as a part of error response + if parts[0] != "" && parts[0][0] == '-' { + parts[0] = parts[0][1:] + } + var final_msg string + if len(parts) == 2 { + final_msg = fmt.Sprintf("%s %s", parts[0], parts[1]) + } else { + final_msg = fmt.Sprintf("ERR %s", parts[0]) + } + res.RawSetString("err", lua.LString(final_msg)) + l.Push(res) + return 1 + }, + "log": func(l *lua.LState) int { + level := l.CheckInt(1) + msg := l.CheckString(2) + _, _ = level, msg + // do nothing by default. To see logs uncomment: + // fmt.Printf("%v: %v", level, msg) + return 0 + }, + "status_reply": func(l *lua.LState) int { + v := l.Get(1) + msg, ok := v.(lua.LString) + if !ok { + l.Error(lua.LString("wrong number or type of arguments"), 1) + return 0 + } + res := luaStatusReply(string(msg)) + l.Push(res) + return 1 + }, + "sha1hex": func(l *lua.LState) int { + top := l.GetTop() + if top != 1 { + l.Error(lua.LString("wrong number of arguments"), 1) + return 0 + } + msg := lua.LVAsString(l.Get(1)) + l.Push(lua.LString(sha1Hex(msg))) + return 1 + }, + "replicate_commands": func(l *lua.LState) int { + // always succeeds since 7.0.0 + l.Push(lua.LTrue) + return 1 + }, + "set_repl": func(l *lua.LState) int { + top := l.GetTop() + if top != 1 { + l.Error(lua.LString("wrong number of arguments"), 1) + return 0 + } + // ignored + return 1 + }, + "setresp": func(l *lua.LState) int { + level := l.CheckInt(1) + toresp3 := false + switch level { + case 2: + toresp3 = false + case 3: + toresp3 = true + default: + l.Error(lua.LString("RESP version must be 2 or 3"), 1) + return 0 + } + c.SwitchResp3 = &toresp3 + return 0 + }, + }, luaRedisConstants +} + +func luaToRedis(l *lua.LState, c *server.Peer, value lua.LValue) { + if value == nil { + c.WriteNull() + return + } + + switch t := value.(type) { + case *lua.LNilType: + c.WriteNull() + case lua.LBool: + if lua.LVAsBool(value) { + c.WriteInt(1) + } else { + c.WriteNull() + } + case lua.LNumber: + c.WriteInt(int(lua.LVAsNumber(value))) + case lua.LString: + s := lua.LVAsString(value) + c.WriteBulk(s) + case *lua.LTable: + // special case for tables with an 'err' or 'ok' field + // note: according to the docs this only counts when 'err' or 'ok' is + // the only field. + if s := t.RawGetString("err"); s.Type() != lua.LTNil { + c.WriteError(s.String()) + return + } + if s := t.RawGetString("ok"); s.Type() != lua.LTNil { + c.WriteInline(s.String()) + return + } + + result := []lua.LValue{} + for j := 1; true; j++ { + val := l.GetTable(value, lua.LNumber(j)) + if val == nil { + result = append(result, val) + continue + } + + if val.Type() == lua.LTNil { + break + } + + result = append(result, val) + } + + c.WriteLen(len(result)) + for _, r := range result { + luaToRedis(l, c, r) + } + default: + panic(fmt.Sprintf("wat: %T", t)) + } +} + +func redisToLua(l *lua.LState, res []interface{}) *lua.LTable { + rettb := l.NewTable() + for _, e := range res { + var v lua.LValue + if e == nil { + v = lua.LFalse + } else { + switch et := e.(type) { + case int: + v = lua.LNumber(et) + case int64: + v = lua.LNumber(et) + case []uint8: + v = lua.LString(string(et)) + case []interface{}: + v = redisToLua(l, et) + case string: + v = lua.LString(et) + default: + // TODO: oops? + v = lua.LString(e.(string)) + } + } + l.RawSet(rettb, lua.LNumber(rettb.Len()+1), v) + } + return rettb +} + +func luaStatusReply(msg string) *lua.LTable { + tab := &lua.LTable{} + tab.RawSetString("ok", lua.LString(msg)) + return tab +} + +// Our very minimal "os." lua lib. +func mkLuaOS() map[string]lua.LGFunction { + return map[string]lua.LGFunction{ + // > Returns an approximation of the amount in seconds of CPU time used by the program + "clock": func(l *lua.LState) int { + l.Push(lua.LNumber(42)) + return 1 + }, + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/LICENSE b/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/LICENSE new file mode 100644 index 00000000..6243b617 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/LICENSE @@ -0,0 +1,24 @@ +This package is a mechanical translation of the reference C++ code for +MetroHash, available at https://github.com/jandrewrogers/MetroHash + +The MIT License (MIT) + +Copyright (c) 2016 Damian Gryski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/README.md b/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/README.md new file mode 100644 index 00000000..07e4ee9f --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/README.md @@ -0,0 +1 @@ +This is a partial copy of github.com/dgryski/go-metro. \ No newline at end of file diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/metro64.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/metro64.go new file mode 100644 index 00000000..5b3db9a9 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/metro/metro64.go @@ -0,0 +1,87 @@ +package metro + +import "encoding/binary" + +func Hash64(buffer []byte, seed uint64) uint64 { + + const ( + k0 = 0xD6D018F5 + k1 = 0xA2AA033B + k2 = 0x62992FC1 + k3 = 0x30BC5B29 + ) + + ptr := buffer + + hash := (seed + k2) * k0 + + if len(ptr) >= 32 { + v := [4]uint64{hash, hash, hash, hash} + + for len(ptr) >= 32 { + v[0] += binary.LittleEndian.Uint64(ptr[:8]) * k0 + v[0] = rotate_right(v[0], 29) + v[2] + v[1] += binary.LittleEndian.Uint64(ptr[8:16]) * k1 + v[1] = rotate_right(v[1], 29) + v[3] + v[2] += binary.LittleEndian.Uint64(ptr[16:24]) * k2 + v[2] = rotate_right(v[2], 29) + v[0] + v[3] += binary.LittleEndian.Uint64(ptr[24:32]) * k3 + v[3] = rotate_right(v[3], 29) + v[1] + ptr = ptr[32:] + } + + v[2] ^= rotate_right(((v[0]+v[3])*k0)+v[1], 37) * k1 + v[3] ^= rotate_right(((v[1]+v[2])*k1)+v[0], 37) * k0 + v[0] ^= rotate_right(((v[0]+v[2])*k0)+v[3], 37) * k1 + v[1] ^= rotate_right(((v[1]+v[3])*k1)+v[2], 37) * k0 + hash += v[0] ^ v[1] + } + + if len(ptr) >= 16 { + v0 := hash + (binary.LittleEndian.Uint64(ptr[:8]) * k2) + v0 = rotate_right(v0, 29) * k3 + v1 := hash + (binary.LittleEndian.Uint64(ptr[8:16]) * k2) + v1 = rotate_right(v1, 29) * k3 + v0 ^= rotate_right(v0*k0, 21) + v1 + v1 ^= rotate_right(v1*k3, 21) + v0 + hash += v1 + ptr = ptr[16:] + } + + if len(ptr) >= 8 { + hash += binary.LittleEndian.Uint64(ptr[:8]) * k3 + ptr = ptr[8:] + hash ^= rotate_right(hash, 55) * k1 + } + + if len(ptr) >= 4 { + hash += uint64(binary.LittleEndian.Uint32(ptr[:4])) * k3 + hash ^= rotate_right(hash, 26) * k1 + ptr = ptr[4:] + } + + if len(ptr) >= 2 { + hash += uint64(binary.LittleEndian.Uint16(ptr[:2])) * k3 + ptr = ptr[2:] + hash ^= rotate_right(hash, 48) * k1 + } + + if len(ptr) >= 1 { + hash += uint64(ptr[0]) * k3 + hash ^= rotate_right(hash, 37) * k1 + } + + hash ^= rotate_right(hash, 28) + hash *= k0 + hash ^= rotate_right(hash, 29) + + return hash +} + +func Hash64Str(buffer string, seed uint64) uint64 { + return Hash64([]byte(buffer), seed) +} + +func rotate_right(v uint64, k uint) uint64 { + return (v >> k) | (v << (64 - k)) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/miniredis.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/miniredis.go new file mode 100644 index 00000000..7e65f163 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/miniredis.go @@ -0,0 +1,792 @@ +// Package miniredis is a pure Go Redis test server, for use in Go unittests. +// There are no dependencies on system binaries, and every server you start +// will be empty. +// +// import "github.com/alicebob/miniredis/v2" +// +// Start a server with `s := miniredis.RunT(t)`, it'll be shutdown via a t.Cleanup(). +// Or do everything manual: `s, err := miniredis.Run(); defer s.Close()` +// +// Point your Redis client to `s.Addr()` or `s.Host(), s.Port()`. +// +// Set keys directly via s.Set(...) and similar commands, or use a Redis client. +// +// For direct use you can select a Redis database with either `s.Select(12); +// s.Get("foo")` or `s.DB(12).Get("foo")`. +package miniredis + +import ( + "context" + "crypto/tls" + "fmt" + "math/rand" + "strconv" + "strings" + "sync" + "time" + + "github.com/alicebob/miniredis/v2/proto" + "github.com/alicebob/miniredis/v2/server" +) + +var DumpMaxLineLen = 60 + +type hashKey map[string]string +type listKey []string +type setKey map[string]struct{} + +// RedisDB holds a single (numbered) Redis database. +type RedisDB struct { + master *Miniredis // pointer to the lock in Miniredis + id int // db id + keys map[string]string // Master map of keys with their type + stringKeys map[string]string // GET/SET &c. keys + hashKeys map[string]hashKey // MGET/MSET &c. keys + listKeys map[string]listKey // LPUSH &c. keys + setKeys map[string]setKey // SADD &c. keys + hllKeys map[string]*hll // PFADD &c. keys + sortedsetKeys map[string]sortedSet // ZADD &c. keys + streamKeys map[string]*streamKey // XADD &c. keys + ttl map[string]time.Duration // effective TTL values + hashTTLs map[string]map[string]time.Duration // Hash TTL values + lru map[string]time.Time // last recently used ( read or written to ) + keyVersion map[string]uint // used to watch values +} + +// Miniredis is a Redis server implementation. +type Miniredis struct { + sync.Mutex + srv *server.Server + port int + passwords map[string]string // username password + dbs map[int]*RedisDB + selectedDB int // DB id used in the direct Get(), Set() &c. + scripts map[string]string // sha1 -> lua src + signal *sync.Cond + now time.Time // time.Now() if not set. + subscribers map[*Subscriber]struct{} + rand *rand.Rand + Ctx context.Context + CtxCancel context.CancelFunc +} + +type txCmd func(*server.Peer, *connCtx) + +// database id + key combo +type dbKey struct { + db int + key string +} + +// connCtx has all state for a single connection. +// (this struct was named before context.Context existed) +type connCtx struct { + selectedDB int // selected DB + authenticated bool // auth enabled and a valid AUTH seen + transaction []txCmd // transaction callbacks. Or nil. + dirtyTransaction bool // any error during QUEUEing + watch map[dbKey]uint // WATCHed keys + subscriber *Subscriber // client is in PUBSUB mode if not nil + nested bool // this is called via Lua + nestedSHA string // set to the SHA of the nesting function +} + +// NewMiniRedis makes a new, non-started, Miniredis object. +func NewMiniRedis() *Miniredis { + m := Miniredis{ + dbs: map[int]*RedisDB{}, + scripts: map[string]string{}, + subscribers: map[*Subscriber]struct{}{}, + } + m.Ctx, m.CtxCancel = context.WithCancel(context.Background()) + m.signal = sync.NewCond(&m) + return &m +} + +func newRedisDB(id int, m *Miniredis) RedisDB { + return RedisDB{ + id: id, + master: m, + keys: map[string]string{}, + lru: map[string]time.Time{}, + stringKeys: map[string]string{}, + hashKeys: map[string]hashKey{}, + listKeys: map[string]listKey{}, + setKeys: map[string]setKey{}, + hllKeys: map[string]*hll{}, + sortedsetKeys: map[string]sortedSet{}, + streamKeys: map[string]*streamKey{}, + ttl: map[string]time.Duration{}, + hashTTLs: make(map[string]map[string]time.Duration), + keyVersion: map[string]uint{}, + } +} + +// Run creates and Start()s a Miniredis. +func Run() (*Miniredis, error) { + m := NewMiniRedis() + return m, m.Start() +} + +// Run creates and Start()s a Miniredis, TLS version. +func RunTLS(cfg *tls.Config) (*Miniredis, error) { + m := NewMiniRedis() + return m, m.StartTLS(cfg) +} + +// Tester is a minimal version of a testing.T +type Tester interface { + Fatalf(string, ...interface{}) + Cleanup(func()) + Logf(format string, args ...interface{}) +} + +// RunT start a new miniredis, pass it a testing.T. It also registers the cleanup after your test is done. +func RunT(t Tester) *Miniredis { + m := NewMiniRedis() + if err := m.Start(); err != nil { + t.Fatalf("could not start miniredis: %s", err) + // not reached + } + t.Cleanup(m.Close) + return m +} + +func runWithClient(t Tester) (*Miniredis, *proto.Client) { + m := RunT(t) + + c, err := proto.Dial(m.Addr()) + if err != nil { + t.Fatalf("could not connect to miniredis: %s", err) + } + t.Cleanup(func() { + if err = c.Close(); err != nil { + t.Logf("error closing connection to miniredis: %s", err) + } + }) + + return m, c +} + +// Start starts a server. It listens on a random port on localhost. See also +// Addr(). +func (m *Miniredis) Start() error { + s, err := server.NewServer(fmt.Sprintf("127.0.0.1:%d", m.port)) + if err != nil { + return err + } + return m.start(s) +} + +// Start starts a server, TLS version. +func (m *Miniredis) StartTLS(cfg *tls.Config) error { + s, err := server.NewServerTLS(fmt.Sprintf("127.0.0.1:%d", m.port), cfg) + if err != nil { + return err + } + return m.start(s) +} + +// StartAddr runs miniredis with a given addr. Examples: "127.0.0.1:6379", +// ":6379", or "127.0.0.1:0" +func (m *Miniredis) StartAddr(addr string) error { + s, err := server.NewServer(addr) + if err != nil { + return err + } + return m.start(s) +} + +// StartAddrTLS runs miniredis with a given addr, TLS version. +func (m *Miniredis) StartAddrTLS(addr string, cfg *tls.Config) error { + s, err := server.NewServerTLS(addr, cfg) + if err != nil { + return err + } + return m.start(s) +} + +func (m *Miniredis) start(s *server.Server) error { + m.Lock() + defer m.Unlock() + m.srv = s + m.port = s.Addr().Port + + commandsConnection(m) + commandsGeneric(m) + commandsServer(m) + commandsString(m) + commandsHash(m) + commandsList(m) + commandsPubsub(m) + commandsSet(m) + commandsSortedSet(m) + commandsStream(m) + commandsTransaction(m) + commandsScripting(m) + commandsGeo(m) + commandsCluster(m) + commandsHll(m) + commandsClient(m) + commandsObject(m) + + return nil +} + +// Restart restarts a Close()d server on the same port. Values will be +// preserved. +func (m *Miniredis) Restart() error { + return m.Start() +} + +// Close shuts down a Miniredis. +func (m *Miniredis) Close() { + m.Lock() + + if m.srv == nil { + m.Unlock() + return + } + srv := m.srv + m.srv = nil + m.CtxCancel() + m.Unlock() + + // the OnDisconnect callbacks can lock m, so run Close() outside the lock. + srv.Close() + +} + +// RequireAuth makes every connection need to AUTH first. This is the old 'AUTH [password] command. +// Remove it by setting an empty string. +func (m *Miniredis) RequireAuth(pw string) { + m.RequireUserAuth("default", pw) +} + +// Add a username/password, for use with 'AUTH [username] [password]'. +// There are currently no access controls for commands implemented. +// Disable access for the user with an empty password. +func (m *Miniredis) RequireUserAuth(username, pw string) { + m.Lock() + defer m.Unlock() + if m.passwords == nil { + m.passwords = map[string]string{} + } + if pw == "" { + delete(m.passwords, username) + return + } + m.passwords[username] = pw +} + +// DB returns a DB by ID. +func (m *Miniredis) DB(i int) *RedisDB { + m.Lock() + defer m.Unlock() + return m.db(i) +} + +// get DB. No locks! +func (m *Miniredis) db(i int) *RedisDB { + if db, ok := m.dbs[i]; ok { + return db + } + db := newRedisDB(i, m) // main miniredis has our mutex. + m.dbs[i] = &db + return &db +} + +// SwapDB swaps DBs by IDs. +func (m *Miniredis) SwapDB(i, j int) bool { + m.Lock() + defer m.Unlock() + return m.swapDB(i, j) +} + +// swap DB. No locks! +func (m *Miniredis) swapDB(i, j int) bool { + db1 := m.db(i) + db2 := m.db(j) + + db1.id = j + db2.id = i + + m.dbs[i] = db2 + m.dbs[j] = db1 + + return true +} + +// Addr returns '127.0.0.1:12345'. Can be given to a Dial(). See also Host() +// and Port(), which return the same things. +func (m *Miniredis) Addr() string { + m.Lock() + defer m.Unlock() + return m.srv.Addr().String() +} + +// Host returns the host part of Addr(). +func (m *Miniredis) Host() string { + m.Lock() + defer m.Unlock() + return m.srv.Addr().IP.String() +} + +// Port returns the (random) port part of Addr(). +func (m *Miniredis) Port() string { + m.Lock() + defer m.Unlock() + return strconv.Itoa(m.srv.Addr().Port) +} + +// CommandCount returns the number of processed commands. +func (m *Miniredis) CommandCount() int { + m.Lock() + defer m.Unlock() + return int(m.srv.TotalCommands()) +} + +// CurrentConnectionCount returns the number of currently connected clients. +func (m *Miniredis) CurrentConnectionCount() int { + m.Lock() + defer m.Unlock() + return m.srv.ClientsLen() +} + +// TotalConnectionCount returns the number of client connections since server start. +func (m *Miniredis) TotalConnectionCount() int { + m.Lock() + defer m.Unlock() + return int(m.srv.TotalConnections()) +} + +// FastForward decreases all TTLs by the given duration. All TTLs <= 0 will be +// expired. +func (m *Miniredis) FastForward(duration time.Duration) { + m.Lock() + defer m.Unlock() + for _, db := range m.dbs { + db.fastForward(duration) + } +} + +// Server returns the underlying server to allow custom commands to be implemented +func (m *Miniredis) Server() *server.Server { + return m.srv +} + +// IsReadOnlyCommand checks if a command is marked as read-only +func (m *Miniredis) IsReadOnlyCommand(cmd string) bool { + if m.srv == nil { + return false + } + return m.srv.IsReadOnlyCommand(cmd) +} + +// Dump returns a text version of the selected DB, usable for debugging. +// +// Dump limits the maximum length of each key:value to "DumpMaxLineLen" characters. +// To increase that, call something like: +// +// miniredis.DumpMaxLineLen = 1024 +// mr, _ = miniredis.Run() +// mr.Dump() +func (m *Miniredis) Dump() string { + m.Lock() + defer m.Unlock() + + var ( + maxLen = DumpMaxLineLen + indent = " " + db = m.db(m.selectedDB) + r = "" + v = func(s string) string { + suffix := "" + if len(s) > maxLen { + suffix = fmt.Sprintf("...(%d)", len(s)) + s = s[:maxLen-len(suffix)] + } + return fmt.Sprintf("%q%s", s, suffix) + } + ) + + for _, k := range db.allKeys() { + r += fmt.Sprintf("- %s\n", k) + t := db.t(k) + switch t { + case keyTypeString: + r += fmt.Sprintf("%s%s\n", indent, v(db.stringKeys[k])) + case keyTypeHash: + for _, hk := range db.hashFields(k) { + r += fmt.Sprintf("%s%s: %s\n", indent, hk, v(db.hashGet(k, hk))) + } + case keyTypeList: + for _, lk := range db.listKeys[k] { + r += fmt.Sprintf("%s%s\n", indent, v(lk)) + } + case keyTypeSet: + for _, mk := range db.setMembers(k) { + r += fmt.Sprintf("%s%s\n", indent, v(mk)) + } + case keyTypeSortedSet: + for _, el := range db.ssetElements(k) { + r += fmt.Sprintf("%s%f: %s\n", indent, el.score, v(el.member)) + } + case keyTypeStream: + for _, entry := range db.streamKeys[k].entries { + r += fmt.Sprintf("%s%s\n", indent, entry.ID) + ev := entry.Values + for i := 0; i < len(ev)/2; i++ { + r += fmt.Sprintf("%s%s%s: %s\n", indent, indent, v(ev[2*i]), v(ev[2*i+1])) + } + } + case keyTypeHll: + for _, entry := range db.hllKeys { + r += fmt.Sprintf("%s%s\n", indent, v(string(entry.Bytes()))) + } + default: + r += fmt.Sprintf("%s(a %s, fixme!)\n", indent, t) + } + } + return r +} + +// SetTime sets the time against which EXPIREAT values are compared, and the +// time used in stream entry IDs. Will use time.Now() if this is not set. +func (m *Miniredis) SetTime(t time.Time) { + m.Lock() + defer m.Unlock() + m.now = t +} + +// make every command return this message. For example: +// +// LOADING Redis is loading the dataset in memory +// MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'. +// +// Clear it with an empty string. Don't add newlines. +func (m *Miniredis) SetError(msg string) { + cb := server.Hook(nil) + if msg != "" { + cb = func(c *server.Peer, cmd string, args ...string) bool { + c.WriteError(msg) + return true + } + } + m.srv.SetPreHook(cb) +} + +type argRequirements struct { + minimum int + maximum *int +} + +func atLeast(n int) argRequirements { + return argRequirements{n, nil} +} + +func between(a int, b int) argRequirements { + return argRequirements{a, &b} +} + +func exactly(n int) argRequirements { + return argRequirements{n, &n} +} + +// isValidCMD returns true if command is valid and can be executed. +func (m *Miniredis) isValidCMD(c *server.Peer, cmd string, args []string, argReqs argRequirements) bool { + if len(args) < argReqs.minimum || (argReqs.maximum != nil && len(args) > *argReqs.maximum) { + setDirty(c) + c.WriteError(errWrongNumber(cmd)) + return false + } + + if !m.handleAuth(c) { + return false + } + if m.checkPubsub(c, cmd) { + return false + } + + return true +} + +// handleAuth returns false if connection has no access. It sends the reply. +func (m *Miniredis) handleAuth(c *server.Peer) bool { + if getCtx(c).nested { + return true + } + + m.Lock() + defer m.Unlock() + if len(m.passwords) == 0 { + return true + } + if !getCtx(c).authenticated { + c.WriteError("NOAUTH Authentication required.") + return false + } + return true +} + +// handlePubsub sends an error to the user if the connection is in PUBSUB mode. +// It'll return true if it did. +func (m *Miniredis) checkPubsub(c *server.Peer, cmd string) bool { + if getCtx(c).nested { + return false + } + + m.Lock() + defer m.Unlock() + + ctx := getCtx(c) + if ctx.subscriber == nil { + return false + } + + prefix := "ERR " + if strings.ToLower(cmd) == "exec" { + prefix = "EXECABORT Transaction discarded because of: " + } + c.WriteError(fmt.Sprintf( + "%sCan't execute '%s': only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT are allowed in this context", + prefix, + strings.ToLower(cmd), + )) + return true +} + +func getCtx(c *server.Peer) *connCtx { + if c.Ctx == nil { + c.Ctx = &connCtx{} + } + return c.Ctx.(*connCtx) +} + +func startTx(ctx *connCtx) { + ctx.transaction = []txCmd{} + ctx.dirtyTransaction = false +} + +func stopTx(ctx *connCtx) { + ctx.transaction = nil + unwatch(ctx) +} + +func inTx(ctx *connCtx) bool { + return ctx.transaction != nil +} + +func addTxCmd(ctx *connCtx, cb txCmd) { + ctx.transaction = append(ctx.transaction, cb) +} + +func watch(db *RedisDB, ctx *connCtx, key string) { + if ctx.watch == nil { + ctx.watch = map[dbKey]uint{} + } + ctx.watch[dbKey{db: db.id, key: key}] = db.keyVersion[key] // Can be 0. +} + +func unwatch(ctx *connCtx) { + ctx.watch = nil +} + +// setDirty can be called even when not in an tx. Is an no-op then. +func setDirty(c *server.Peer) { + if c.Ctx == nil { + // No transaction. Not relevant. + return + } + getCtx(c).dirtyTransaction = true +} + +func (m *Miniredis) addSubscriber(s *Subscriber) { + m.subscribers[s] = struct{}{} +} + +// closes and remove the subscriber. +func (m *Miniredis) removeSubscriber(s *Subscriber) { + _, ok := m.subscribers[s] + delete(m.subscribers, s) + if ok { + s.Close() + } +} + +func (m *Miniredis) publish(c, msg string) int { + n := 0 + for s := range m.subscribers { + n += s.Publish(c, msg) + } + return n +} + +// enter 'subscribed state', or return the existing one. +func (m *Miniredis) subscribedState(c *server.Peer) *Subscriber { + ctx := getCtx(c) + sub := ctx.subscriber + if sub != nil { + return sub + } + + sub = newSubscriber() + m.addSubscriber(sub) + + c.OnDisconnect(func() { + m.Lock() + m.removeSubscriber(sub) + m.Unlock() + }) + + ctx.subscriber = sub + + go monitorPublish(c, sub.publish) + go monitorPpublish(c, sub.ppublish) + + return sub +} + +// whenever the p?sub count drops to 0 subscribed state should be stopped, and +// all redis commands are allowed again. +func endSubscriber(m *Miniredis, c *server.Peer) { + ctx := getCtx(c) + if sub := ctx.subscriber; sub != nil { + m.removeSubscriber(sub) // will Close() the sub + } + ctx.subscriber = nil +} + +// Start a new pubsub subscriber. It can (un) subscribe to channels and +// patterns, and has a channel to get published messages. Close it with +// Close(). +// Does not close itself when there are no subscriptions left. +func (m *Miniredis) NewSubscriber() *Subscriber { + sub := newSubscriber() + + m.Lock() + m.addSubscriber(sub) + m.Unlock() + + return sub +} + +func (m *Miniredis) allSubscribers() []*Subscriber { + var subs []*Subscriber + for s := range m.subscribers { + subs = append(subs, s) + } + return subs +} + +func (m *Miniredis) Seed(seed int) { + m.Lock() + defer m.Unlock() + + // m.rand is not safe for concurrent use. + m.rand = rand.New(rand.NewSource(int64(seed))) +} + +func (m *Miniredis) randIntn(n int) int { + if m.rand == nil { + return rand.Intn(n) + } + return m.rand.Intn(n) +} + +// shuffle shuffles a list of strings. Kinda. +func (m *Miniredis) shuffle(l []string) { + for range l { + i := m.randIntn(len(l)) + j := m.randIntn(len(l)) + l[i], l[j] = l[j], l[i] + } +} + +func (m *Miniredis) effectiveNow() time.Time { + if !m.now.IsZero() { + return m.now + } + return time.Now().UTC() +} + +// convert a unixtimestamp to a duration, to use an absolute time as TTL. +// d can be either time.Second or time.Millisecond. +func (m *Miniredis) at(i int, d time.Duration) time.Duration { + var ts time.Time + switch d { + case time.Millisecond: + ts = time.Unix(int64(i/1000), 1000000*int64(i%1000)) + case time.Second: + ts = time.Unix(int64(i), 0) + default: + panic("invalid time unit (d). Fixme!") + } + now := m.effectiveNow() + return ts.Sub(now) +} + +// copy does not mind if dst already exists. +func (m *Miniredis) copy( + srcDB *RedisDB, src string, + destDB *RedisDB, dst string, +) error { + if !srcDB.exists(src) { + return ErrKeyNotFound + } + + switch srcDB.t(src) { + case keyTypeString: + destDB.stringKeys[dst] = srcDB.stringKeys[src] + case keyTypeHash: + destDB.hashKeys[dst] = copyHashKey(srcDB.hashKeys[src]) + case keyTypeList: + destDB.listKeys[dst] = copyListKey(srcDB.listKeys[src]) + case keyTypeSet: + destDB.setKeys[dst] = copySetKey(srcDB.setKeys[src]) + case keyTypeSortedSet: + destDB.sortedsetKeys[dst] = copySortedSet(srcDB.sortedsetKeys[src]) + case keyTypeStream: + destDB.streamKeys[dst] = srcDB.streamKeys[src].copy() + case keyTypeHll: + destDB.hllKeys[dst] = srcDB.hllKeys[src].copy() + default: + panic("missing case") + } + destDB.keys[dst] = srcDB.keys[src] + destDB.incr(dst) + if v, ok := srcDB.ttl[src]; ok { + destDB.ttl[dst] = v + } + return nil +} + +func copyHashKey(orig hashKey) hashKey { + cpy := hashKey{} + for k, v := range orig { + cpy[k] = v + } + return cpy +} + +func copyListKey(orig listKey) listKey { + cpy := make(listKey, len(orig)) + copy(cpy, orig) + return cpy +} + +func copySetKey(orig setKey) setKey { + cpy := setKey{} + for k, v := range orig { + cpy[k] = v + } + return cpy +} + +func copySortedSet(orig sortedSet) sortedSet { + cpy := sortedSet{} + for k, v := range orig { + cpy[k] = v + } + return cpy +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/opts.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/opts.go new file mode 100644 index 00000000..5b29c78c --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/opts.go @@ -0,0 +1,60 @@ +package miniredis + +import ( + "errors" + "math" + "strconv" + "time" + + "github.com/alicebob/miniredis/v2/server" +) + +// optInt parses an int option in a command. +// Writes "invalid integer" error to c if it's not a valid integer. Returns +// whether or not things were okay. +func optInt(c *server.Peer, src string, dest *int) bool { + return optIntErr(c, src, dest, msgInvalidInt) +} + +func optIntErr(c *server.Peer, src string, dest *int, errMsg string) bool { + n, err := strconv.Atoi(src) + if err != nil { + setDirty(c) + c.WriteError(errMsg) + return false + } + *dest = n + return true +} + +// optIntSimple sets dest or returns an error +func optIntSimple(src string, dest *int) error { + n, err := strconv.Atoi(src) + if err != nil { + return errors.New(msgInvalidInt) + } + *dest = n + return nil +} + +func optDuration(c *server.Peer, src string, dest *time.Duration) bool { + n, err := strconv.ParseFloat(src, 64) + if err != nil { + setDirty(c) + c.WriteError(msgInvalidTimeout) + return false + } + if n < 0 { + setDirty(c) + c.WriteError(msgTimeoutNegative) + return false + } + if math.IsInf(n, 0) { + setDirty(c) + c.WriteError(msgTimeoutIsOutOfRange) + return false + } + + *dest = time.Duration(n*1_000_000) * time.Microsecond + return true +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/Makefile b/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/Makefile new file mode 100644 index 00000000..b9ef3949 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/Makefile @@ -0,0 +1,2 @@ +test: + go test diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/client.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/client.go new file mode 100644 index 00000000..92f57baf --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/client.go @@ -0,0 +1,60 @@ +package proto + +import ( + "bufio" + "crypto/tls" + "net" +) + +type Client struct { + c net.Conn + r *bufio.Reader +} + +func Dial(addr string) (*Client, error) { + c, err := net.Dial("tcp", addr) + if err != nil { + return nil, err + } + + return &Client{ + c: c, + r: bufio.NewReader(c), + }, nil +} + +func DialTLS(addr string, cfg *tls.Config) (*Client, error) { + c, err := tls.Dial("tcp", addr, cfg) + if err != nil { + return nil, err + } + + return &Client{ + c: c, + r: bufio.NewReader(c), + }, nil +} + +func (c *Client) Close() error { + return c.c.Close() +} + +func (c *Client) Do(cmd ...string) (string, error) { + if err := Write(c.c, cmd); err != nil { + return "", err + } + return Read(c.r) +} + +func (c *Client) Read() (string, error) { + return Read(c.r) +} + +// Do() + ReadStrings() +func (c *Client) DoStrings(cmd ...string) ([]string, error) { + res, err := c.Do(cmd...) + if err != nil { + return nil, err + } + return ReadStrings(res) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/proto.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/proto.go new file mode 100644 index 00000000..e378faf1 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/proto.go @@ -0,0 +1,288 @@ +package proto + +import ( + "bufio" + "errors" + "fmt" + "io" + "strconv" + "strings" +) + +var ( + ErrProtocol = errors.New("unsupported protocol") + ErrUnexpected = errors.New("not what you asked for") +) + +func readLine(r *bufio.Reader) (string, error) { + line, err := r.ReadString('\n') + if err != nil { + return "", err + } + if len(line) < 3 { + return "", ErrProtocol + } + return line, nil +} + +// Read an array, with all elements are the raw redis commands +// Also reads sets and maps. +func ReadArray(b string) ([]string, error) { + r := bufio.NewReader(strings.NewReader(b)) + line, err := readLine(r) + if err != nil { + return nil, err + } + + elems := 0 + switch line[0] { + default: + return nil, ErrUnexpected + case '*', '>', '~': + // *: array + // >: push data + // ~: set + length, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return nil, err + } + elems = length + case '%': + // we also read maps. + length, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return nil, err + } + elems = length * 2 + } + + var res []string + for i := 0; i < elems; i++ { + next, err := Read(r) + if err != nil { + return nil, err + } + res = append(res, next) + } + return res, nil +} + +func ReadString(b string) (string, error) { + r := bufio.NewReader(strings.NewReader(b)) + line, err := readLine(r) + if err != nil { + return "", err + } + + switch line[0] { + default: + return "", ErrUnexpected + case '$': + // bulk strings are: `$5\r\nhello\r\n` + length, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return "", err + } + if length < 0 { + // -1 is a nil response + return line, nil + } + var ( + buf = make([]byte, length+2) + pos = 0 + ) + for pos < length+2 { + n, err := r.Read(buf[pos:]) + if err != nil { + return "", err + } + pos += n + } + return string(buf[:len(buf)-2]), nil + } +} + +func readInline(b string) (string, error) { + if len(b) < 3 { + return "", ErrUnexpected + } + return b[1 : len(b)-2], nil +} + +func ReadError(b string) (string, error) { + if len(b) < 1 { + return "", ErrUnexpected + } + + switch b[0] { + default: + return "", ErrUnexpected + case '-': + return readInline(b) + } +} + +func ReadStrings(b string) ([]string, error) { + elems, err := ReadArray(b) + if err != nil { + return nil, err + } + var res []string + for _, e := range elems { + s, err := ReadString(e) + if err != nil { + return nil, err + } + res = append(res, s) + } + return res, nil +} + +// Read a single command, returning it raw. Used to read replies from redis. +// Understands RESP3 proto. +func Read(r *bufio.Reader) (string, error) { + line, err := readLine(r) + if err != nil { + return "", err + } + + switch line[0] { + default: + return "", ErrProtocol + case '+', '-', ':', ',', '_': + // +: inline string + // -: errors + // :: integer + // ,: float + // _: null + // Simple line based replies. + return line, nil + case '$': + // bulk strings are: `$5\r\nhello\r\n` + length, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return "", err + } + if length < 0 { + // -1 is a nil response + return line, nil + } + var ( + buf = make([]byte, length+2) + pos = 0 + ) + for pos < length+2 { + n, err := r.Read(buf[pos:]) + if err != nil { + return "", err + } + pos += n + } + return line + string(buf), nil + case '*', '>', '~': + // arrays are: `*6\r\n...` + // pushdata is: `>6\r\n...` + // sets are: `~6\r\n...` + length, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return "", err + } + for i := 0; i < length; i++ { + next, err := Read(r) + if err != nil { + return "", err + } + line += next + } + return line, nil + case '%': + // maps are: `%3\r\n...` + length, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return "", err + } + for i := 0; i < length*2; i++ { + next, err := Read(r) + if err != nil { + return "", err + } + line += next + } + return line, nil + } +} + +// Write a command in RESP3 proto. Used to write commands to redis. +// Currently only supports string arrays. +func Write(w io.Writer, cmd []string) error { + if _, err := fmt.Fprintf(w, "*%d\r\n", len(cmd)); err != nil { + return err + } + for _, c := range cmd { + if _, err := fmt.Fprintf(w, "$%d\r\n%s\r\n", len(c), c); err != nil { + return err + } + } + return nil +} + +// Parse into interfaces. `b` must contain exactly a single command (which can be nested). +func Parse(b string) (interface{}, error) { + if len(b) < 1 { + return nil, ErrUnexpected + } + + switch b[0] { + default: + return "", ErrProtocol + case '+': + return readInline(b) + case '-': + e, err := readInline(b) + if err != nil { + return nil, err + } + return errors.New(e), nil + case ':': + e, err := readInline(b) + if err != nil { + return nil, err + } + return strconv.Atoi(e) + case '$': + return ReadString(b) + case '*': + elems, err := ReadArray(b) + if err != nil { + return nil, err + } + var res []interface{} + for _, elem := range elems { + e, err := Parse(elem) + if err != nil { + return nil, err + } + res = append(res, e) + } + return res, nil + case '%': + elems, err := ReadArray(b) + if err != nil { + return nil, err + } + var res = map[interface{}]interface{}{} + for len(elems) > 1 { + key, err := Parse(elems[0]) + if err != nil { + return nil, err + } + value, err := Parse(elems[1]) + if err != nil { + return nil, err + } + res[key] = value + elems = elems[2:] + } + return res, nil + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/types.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/types.go new file mode 100644 index 00000000..0b3b7c9a --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/proto/types.go @@ -0,0 +1,102 @@ +package proto + +import ( + "fmt" + "strings" +) + +// Byte-safe string +func String(s string) string { + return fmt.Sprintf("$%d\r\n%s\r\n", len(s), s) +} + +// Inline string +func Inline(s string) string { + return inline('+', s) +} + +// Error +func Error(s string) string { + return inline('-', s) +} + +func inline(r rune, s string) string { + return fmt.Sprintf("%s%s\r\n", string(r), s) +} + +// Int +func Int(n int) string { + return fmt.Sprintf(":%d\r\n", n) +} + +// Float +func Float(n float64) string { + return fmt.Sprintf(",%g\r\n", n) +} + +const ( + Nil = "$-1\r\n" + NilResp3 = "_\r\n" + NilList = "*-1\r\n" +) + +// Array assembles the args in a list. Args should be raw redis commands. +// Example: Array(String("foo"), String("bar")) +func Array(args ...string) string { + return fmt.Sprintf("*%d\r\n", len(args)) + strings.Join(args, "") +} + +// Push assembles the args for push-data. Args should be raw redis commands. +// Example: Push(String("foo"), String("bar")) +func Push(args ...string) string { + return fmt.Sprintf(">%d\r\n", len(args)) + strings.Join(args, "") +} + +// Strings is a helper to build 1 dimensional string arrays. +func Strings(args ...string) string { + var strings []string + for _, a := range args { + strings = append(strings, String(a)) + } + return Array(strings...) +} + +// Ints is a helper to build 1 dimensional int arrays. +func Ints(args ...int) string { + var ints []string + for _, a := range args { + ints = append(ints, Int(a)) + } + return Array(ints...) +} + +// Map assembles the args in a map. Args should be raw redis commands. +// Must be an even number of arguments. +// Example: Map(String("foo"), String("bar")) +func Map(args ...string) string { + return fmt.Sprintf("%%%d\r\n", len(args)/2) + strings.Join(args, "") +} + +// StringMap is is a wrapper to get a map of (bulk)strings. +func StringMap(args ...string) string { + var strings []string + for _, a := range args { + strings = append(strings, String(a)) + } + return Map(strings...) +} + +// Set assembles the args in a map. Args should be raw redis commands. +// Example: Set(String("foo"), String("bar")) +func Set(args ...string) string { + return fmt.Sprintf("~%d\r\n", len(args)) + strings.Join(args, "") +} + +// StringSet is is a wrapper to get a set of (bulk)strings. +func StringSet(args ...string) string { + var strings []string + for _, a := range args { + strings = append(strings, String(a)) + } + return Set(strings...) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/pubsub.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/pubsub.go new file mode 100644 index 00000000..bb31f80a --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/pubsub.go @@ -0,0 +1,240 @@ +package miniredis + +import ( + "regexp" + "sort" + "sync" + + "github.com/alicebob/miniredis/v2/server" +) + +// PubsubMessage is what gets broadcasted over pubsub channels. +type PubsubMessage struct { + Channel string + Message string +} + +type PubsubPmessage struct { + Pattern string + Channel string + Message string +} + +// Subscriber has the (p)subscriptions. +type Subscriber struct { + publish chan PubsubMessage + ppublish chan PubsubPmessage + channels map[string]struct{} + patterns map[string]*regexp.Regexp + mu sync.Mutex +} + +// Make a new subscriber. The channel is not buffered, so you will need to keep +// reading using Messages(). Use Close() when done, or unsubscribe. +func newSubscriber() *Subscriber { + return &Subscriber{ + publish: make(chan PubsubMessage), + ppublish: make(chan PubsubPmessage), + channels: map[string]struct{}{}, + patterns: map[string]*regexp.Regexp{}, + } +} + +// Close the listening channel +func (s *Subscriber) Close() { + close(s.publish) + close(s.ppublish) +} + +// Count the total number of channels and patterns +func (s *Subscriber) Count() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.count() +} + +func (s *Subscriber) count() int { + return len(s.channels) + len(s.patterns) +} + +// Subscribe to a channel. Returns the total number of (p)subscriptions after +// subscribing. +func (s *Subscriber) Subscribe(c string) int { + s.mu.Lock() + defer s.mu.Unlock() + + s.channels[c] = struct{}{} + return s.count() +} + +// Unsubscribe a channel. Returns the total number of (p)subscriptions after +// unsubscribing. +func (s *Subscriber) Unsubscribe(c string) int { + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.channels, c) + return s.count() +} + +// Subscribe to a pattern. Returns the total number of (p)subscriptions after +// subscribing. +func (s *Subscriber) Psubscribe(pat string) int { + s.mu.Lock() + defer s.mu.Unlock() + + s.patterns[pat] = patternRE(pat) + return s.count() +} + +// Unsubscribe a pattern. Returns the total number of (p)subscriptions after +// unsubscribing. +func (s *Subscriber) Punsubscribe(pat string) int { + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.patterns, pat) + return s.count() +} + +// List all subscribed channels, in alphabetical order +func (s *Subscriber) Channels() []string { + s.mu.Lock() + defer s.mu.Unlock() + + var cs []string + for c := range s.channels { + cs = append(cs, c) + } + sort.Strings(cs) + return cs +} + +// List all subscribed patterns, in alphabetical order +func (s *Subscriber) Patterns() []string { + s.mu.Lock() + defer s.mu.Unlock() + + var ps []string + for p := range s.patterns { + ps = append(ps, p) + } + sort.Strings(ps) + return ps +} + +// Publish a message. Will return return how often we sent the message (can be +// a match for a subscription and for a psubscription. +func (s *Subscriber) Publish(c, msg string) int { + s.mu.Lock() + defer s.mu.Unlock() + + found := 0 + +subs: + for sub := range s.channels { + if sub == c { + s.publish <- PubsubMessage{c, msg} + found++ + break subs + } + } + +pats: + for orig, pat := range s.patterns { + if pat != nil && pat.MatchString(c) { + s.ppublish <- PubsubPmessage{orig, c, msg} + found++ + break pats + } + } + + return found +} + +// The channel to read messages for this subscriber. Only for messages matching +// a SUBSCRIBE. +func (s *Subscriber) Messages() <-chan PubsubMessage { + return s.publish +} + +// The channel to read messages for this subscriber. Only for messages matching +// a PSUBSCRIBE. +func (s *Subscriber) Pmessages() <-chan PubsubPmessage { + return s.ppublish +} + +// List all pubsub channels. If `pat` isn't empty channels names must match the +// pattern. Channels are returned alphabetically. +func activeChannels(subs []*Subscriber, pat string) []string { + channels := map[string]struct{}{} + for _, s := range subs { + for c := range s.channels { + channels[c] = struct{}{} + } + } + + var cpat *regexp.Regexp + if pat != "" { + cpat = patternRE(pat) + } + + var cs []string + for k := range channels { + if cpat != nil && !cpat.MatchString(k) { + continue + } + cs = append(cs, k) + } + sort.Strings(cs) + return cs +} + +// Count all subscribed (not psubscribed) clients for the given channel +// pattern. Channels are returned alphabetically. +func countSubs(subs []*Subscriber, channel string) int { + n := 0 + for _, p := range subs { + for c := range p.channels { + if c == channel { + n++ + break + } + } + } + return n +} + +// Count the total of all client psubscriptions. +func countPsubs(subs []*Subscriber) int { + n := 0 + for _, p := range subs { + n += len(p.patterns) + } + return n +} + +func monitorPublish(conn *server.Peer, msgs <-chan PubsubMessage) { + for msg := range msgs { + conn.Block(func(c *server.Writer) { + c.WritePushLen(3) + c.WriteBulk("message") + c.WriteBulk(msg.Channel) + c.WriteBulk(msg.Message) + c.Flush() + }) + } +} + +func monitorPpublish(conn *server.Peer, msgs <-chan PubsubPmessage) { + for msg := range msgs { + conn.Block(func(c *server.Writer) { + c.WritePushLen(4) + c.WriteBulk("pmessage") + c.WriteBulk(msg.Pattern) + c.WriteBulk(msg.Channel) + c.WriteBulk(msg.Message) + c.Flush() + }) + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/redis.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/redis.go new file mode 100644 index 00000000..eae0e2ff --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/redis.go @@ -0,0 +1,269 @@ +package miniredis + +import ( + "context" + "fmt" + "math" + "math/big" + "strings" + "sync" + "time" + + "github.com/alicebob/miniredis/v2/server" +) + +const ( + keyTypeString = "string" + keyTypeHash = "hash" + keyTypeList = "list" + keyTypeSet = "set" + keyTypeHll = "hll" + keyTypeSortedSet = "zset" + keyTypeStream = "stream" +) + +const ( + msgWrongType = "WRONGTYPE Operation against a key holding the wrong kind of value" + msgNotValidHllValue = "WRONGTYPE Key is not a valid HyperLogLog string value." + msgInvalidInt = "ERR value is not an integer or out of range" + msgIntOverflow = "ERR increment or decrement would overflow" + msgInvalidFloat = "ERR value is not a valid float" + msgInvalidMinMax = "ERR min or max is not a float" + msgInvalidRangeItem = "ERR min or max not valid string range item" + msgInvalidTimeout = "ERR timeout is not a float or out of range" + msgInvalidRange = "ERR value is out of range, must be positive" + msgSyntaxError = "ERR syntax error" + msgKeyNotFound = "ERR no such key" + msgOutOfRange = "ERR index out of range" + msgInvalidCursor = "ERR invalid cursor" + msgXXandNX = "ERR XX and NX options at the same time are not compatible" + msgTimeoutNegative = "ERR timeout is negative" + msgTimeoutIsOutOfRange = "ERR timeout is out of range" + msgInvalidSETime = "ERR invalid expire time in set" + msgInvalidSETEXTime = "ERR invalid expire time in setex" + msgInvalidPSETEXTime = "ERR invalid expire time in psetex" + msgInvalidKeysNumber = "ERR Number of keys can't be greater than number of args" + msgNegativeKeysNumber = "ERR Number of keys can't be negative" + msgFScriptUsage = "ERR unknown subcommand or wrong number of arguments for '%s'. Try SCRIPT HELP." + msgFScriptUsageSimple = "ERR unknown subcommand '%s'. Try SCRIPT HELP." + msgFPubsubUsage = "ERR unknown subcommand or wrong number of arguments for '%s'. Try PUBSUB HELP." + msgFPubsubUsageSimple = "ERR unknown subcommand '%s'. Try PUBSUB HELP." + msgFObjectUsage = "ERR unknown subcommand '%s'. Try OBJECT HELP." + msgScriptFlush = "ERR SCRIPT FLUSH only support SYNC|ASYNC option" + msgSingleElementPair = "ERR INCR option supports a single increment-element pair" + msgGTLTandNX = "ERR GT, LT, and/or NX options at the same time are not compatible" + msgInvalidStreamID = "ERR Invalid stream ID specified as stream command argument" + msgStreamIDTooSmall = "ERR The ID specified in XADD is equal or smaller than the target stream top item" + msgStreamIDZero = "ERR The ID specified in XADD must be greater than 0-0" + msgNoScriptFound = "NOSCRIPT No matching script. Please use EVAL." + msgUnsupportedUnit = "ERR unsupported unit provided. please use M, KM, FT, MI" + msgXreadUnbalanced = "ERR Unbalanced 'xread' list of streams: for each stream key an ID or '$' must be specified." + msgXgroupKeyNotFound = "ERR The XGROUP subcommand requires the key to exist. Note that for CREATE you may want to use the MKSTREAM option to create an empty stream automatically." + msgXtrimInvalidStrategy = "ERR unsupported XTRIM strategy. Please use MAXLEN, MINID" + msgXtrimInvalidMaxLen = "ERR value is not an integer or out of range" + msgXtrimInvalidLimit = "ERR syntax error, LIMIT cannot be used without the special ~ option" + msgDBIndexOutOfRange = "ERR DB index is out of range" + msgLimitCombination = "ERR syntax error, LIMIT is only supported in combination with either BYSCORE or BYLEX" + msgRankIsZero = "ERR RANK can't be zero: use 1 to start from the first match, 2 from the second ... or use negative to start from the end of the list" + msgCountIsNegative = "ERR COUNT can't be negative" + msgMaxLengthIsNegative = "ERR MAXLEN can't be negative" + msgLimitIsNegative = "ERR LIMIT can't be negative" + msgMemorySubcommand = "ERR unknown subcommand '%s'. Try MEMORY HELP." + msgNumFieldsParameter = "ERR The `numfields` parameter must match the number of arguments" + msgNumFieldsInvalid = "ERR Parameter `numFields` should be greater than 0" + msgMandatoryArgument = "ERR Mandatory argument %s is missing or not at the right position" + msgGTandLT = "ERR GT and LT options at the same time are not compatible" + msgNXandXXGTLT = "ERR NX and XX, GT or LT options at the same time are not compatible" +) + +func errWrongNumber(cmd string) string { + return fmt.Sprintf("ERR wrong number of arguments for '%s' command", strings.ToLower(cmd)) +} + +func errLuaParseError(err error) string { + return fmt.Sprintf("ERR Error compiling script (new function): %s", err.Error()) +} + +func errReadgroup(key, group string) error { + return fmt.Errorf("NOGROUP No such key '%s' or consumer group '%s'", key, group) +} + +func errXreadgroup(key, group string) error { + return fmt.Errorf("NOGROUP No such key '%s' or consumer group '%s' in XREADGROUP with GROUP option", key, group) +} + +func msgNotFromScripts(sha string) string { + return fmt.Sprintf("This Redis command is not allowed from script script: %s, &c", sha) +} + +// withTx wraps the non-argument-checking part of command handling code in +// transaction logic. +func withTx( + m *Miniredis, + c *server.Peer, + cb txCmd, +) { + ctx := getCtx(c) + + if ctx.nested { + // this is a call via Lua's .call(). It's already locked. + cb(c, ctx) + m.signal.Broadcast() + return + } + + if inTx(ctx) { + addTxCmd(ctx, cb) + c.WriteInline("QUEUED") + return + } + m.Lock() + cb(c, ctx) + // done, wake up anyone who waits on anything. + m.signal.Broadcast() + m.Unlock() +} + +// blockCmd is executed returns whether it is done +type blockCmd func(*server.Peer, *connCtx) bool + +// blocking keeps trying a command until the callback returns true. Calls +// onTimeout after the timeout (or when we call this in a transaction). +func blocking( + m *Miniredis, + c *server.Peer, + timeout time.Duration, + cb blockCmd, + onTimeout func(*server.Peer), +) { + var ( + ctx = getCtx(c) + ) + if inTx(ctx) { + addTxCmd(ctx, func(c *server.Peer, ctx *connCtx) { + if !cb(c, ctx) { + onTimeout(c) + } + }) + c.WriteInline("QUEUED") + return + } + + localCtx, cancel := context.WithCancel(m.Ctx) + defer cancel() + timedOut := false + if timeout != 0 { + go setCondTimer(localCtx, m.signal, &timedOut, timeout) + } + go func() { + <-localCtx.Done() + m.signal.Broadcast() // main loop might miss this signal + }() + + if !ctx.nested { + // this is a call via Lua's .call(). It's already locked. + m.Lock() + defer m.Unlock() + } + for { + if c.Closed() { + return + } + + if m.Ctx.Err() != nil { + return + } + + done := cb(c, ctx) + if done { + return + } + + if timedOut { + onTimeout(c) + return + } + + m.signal.Wait() + } +} + +func setCondTimer(ctx context.Context, sig *sync.Cond, timedOut *bool, timeout time.Duration) { + dl := time.NewTimer(timeout) + defer dl.Stop() + select { + case <-dl.C: + sig.L.Lock() // for timedOut + *timedOut = true + sig.Broadcast() // main loop might miss this signal + sig.L.Unlock() + case <-ctx.Done(): + } +} + +// formatBig formats a float the way redis does +func formatBig(v *big.Float) string { + // Format with %f and strip trailing 0s. + if v.IsInf() { + return "inf" + } + // if math.IsInf(v, -1) { + // return "-inf" + // } + return stripZeros(fmt.Sprintf("%.17f", v)) +} + +func stripZeros(sv string) string { + for strings.Contains(sv, ".") { + if sv[len(sv)-1] != '0' { + break + } + // Remove trailing 0s. + sv = sv[:len(sv)-1] + // Ends with a '.'. + if sv[len(sv)-1] == '.' { + sv = sv[:len(sv)-1] + break + } + } + return sv +} + +// redisRange gives Go offsets for something l long with start/end in +// Redis semantics. Both start and end can be negative. +// Used for string range and list range things. +// The results can be used as: v[start:end] +// Note that GETRANGE (on a string key) never returns an empty string when end +// is a large negative number. +func redisRange(l, start, end int, stringSymantics bool) (int, int) { + if start < 0 { + start = l + start + if start < 0 { + start = 0 + } + } + if start > l { + start = l + } + + if end < 0 { + end = l + end + if end < 0 { + end = -1 + if stringSymantics { + end = 0 + } + } + } + if end < math.MaxInt32 { + end++ // end argument is inclusive in Redis. + } + if end > l { + end = l + } + + if end < start { + return 0, 0 + } + return start, end +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/Makefile b/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/Makefile new file mode 100644 index 00000000..c82e336f --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/Makefile @@ -0,0 +1,9 @@ +.PHONY: all build test + +all: build test + +build: + go build + +test: + go test diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/cmdmeta.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/cmdmeta.go new file mode 100644 index 00000000..a1be7e15 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/cmdmeta.go @@ -0,0 +1,17 @@ +package server + +// cmdMeta holds metadata about a registered command +type cmdMeta struct { + handler Cmd + readOnly bool +} + +// CmdOption is a function that configures command metadata +type CmdOption func(*cmdMeta) + +// ReadOnlyOption marks a command as read-only +func ReadOnlyOption() CmdOption { + return func(meta *cmdMeta) { + meta.readOnly = true + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/proto.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/proto.go new file mode 100644 index 00000000..f62e1d73 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/proto.go @@ -0,0 +1,157 @@ +package server + +import ( + "bufio" + "errors" + "strconv" +) + +type Simple string + +// ErrProtocol is the general error for unexpected input +var ErrProtocol = errors.New("invalid request") + +// client always sends arrays with bulk strings +func readArray(rd *bufio.Reader) ([]string, error) { + line, err := rd.ReadString('\n') + if err != nil { + return nil, err + } + if len(line) < 3 { + return nil, ErrProtocol + } + + switch line[0] { + default: + return nil, ErrProtocol + case '*': + l, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return nil, err + } + // l can be -1 + var fields []string + for ; l > 0; l-- { + s, err := readString(rd) + if err != nil { + return nil, err + } + fields = append(fields, s) + } + return fields, nil + } +} + +func readString(rd *bufio.Reader) (string, error) { + line, err := rd.ReadString('\n') + if err != nil { + return "", err + } + if len(line) < 3 { + return "", ErrProtocol + } + + switch line[0] { + default: + return "", ErrProtocol + case '+', '-', ':': + // +: simple string + // -: errors + // :: integer + // Simple line based replies. + return string(line[1 : len(line)-2]), nil + case '$': + // bulk strings are: `$5\r\nhello\r\n` + length, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return "", err + } + if length < 0 { + // -1 is a nil response + return "", nil + } + var ( + buf = make([]byte, length+2) + pos = 0 + ) + for pos < length+2 { + n, err := rd.Read(buf[pos:]) + if err != nil { + return "", err + } + pos += n + } + return string(buf[:length]), nil + } +} + +// parse a reply +func ParseReply(rd *bufio.Reader) (interface{}, error) { + line, err := rd.ReadString('\n') + if err != nil { + return nil, err + } + if len(line) < 3 { + return nil, ErrProtocol + } + + switch line[0] { + default: + return nil, ErrProtocol + case '+': + // +: simple string + return Simple(line[1 : len(line)-2]), nil + case '-': + // -: errors + return nil, errors.New(string(line[1 : len(line)-2])) + case ':': + // :: integer + v := line[1 : len(line)-2] + if v == "" { + return 0, nil + } + n, err := strconv.Atoi(v) + if err != nil { + return nil, ErrProtocol + } + return n, nil + case '$': + // bulk strings are: `$5\r\nhello\r\n` + length, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return "", err + } + if length < 0 { + // -1 is a nil response + return nil, nil + } + var ( + buf = make([]byte, length+2) + pos = 0 + ) + for pos < length+2 { + n, err := rd.Read(buf[pos:]) + if err != nil { + return "", err + } + pos += n + } + return string(buf[:length]), nil + case '*': + // array + l, err := strconv.Atoi(line[1 : len(line)-2]) + if err != nil { + return nil, ErrProtocol + } + // l can be -1 + var fields []interface{} + for ; l > 0; l-- { + s, err := ParseReply(rd) + if err != nil { + return nil, err + } + fields = append(fields, s) + } + return fields, nil + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/server.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/server.go new file mode 100644 index 00000000..af36105f --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/server/server.go @@ -0,0 +1,519 @@ +package server + +import ( + "bufio" + "crypto/tls" + "fmt" + "net" + "strings" + "sync" + "unicode" + + "github.com/alicebob/miniredis/v2/fpconv" +) + +func errUnknownCommand(cmd string, args []string) string { + s := fmt.Sprintf("ERR unknown command `%s`, with args beginning with: ", cmd) + if len(args) > 20 { + args = args[:20] + } + for _, a := range args { + s += fmt.Sprintf("`%s`, ", a) + } + return s +} + +// Cmd is what Register expects +type Cmd func(c *Peer, cmd string, args []string) + +type DisconnectHandler func(c *Peer) + +// Hook is can be added to run before every cmd. Return true if the command is done. +type Hook func(*Peer, string, ...string) bool + +// Server is a simple redis server +type Server struct { + l net.Listener + cmds map[string]*cmdMeta + preHook Hook + peers map[net.Conn]struct{} + mu sync.Mutex + wg sync.WaitGroup + infoConns int + infoCmds int +} + +// NewServer makes a server listening on addr. Close with .Close(). +func NewServer(addr string) (*Server, error) { + l, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + return newServer(l), nil +} + +func NewServerTLS(addr string, cfg *tls.Config) (*Server, error) { + l, err := tls.Listen("tcp", addr, cfg) + if err != nil { + return nil, err + } + return newServer(l), nil +} + +func newServer(l net.Listener) *Server { + s := Server{ + cmds: map[string]*cmdMeta{}, + peers: map[net.Conn]struct{}{}, + l: l, + } + + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.serve(l) + + s.mu.Lock() + for c := range s.peers { + c.Close() + } + s.mu.Unlock() + }() + return &s +} + +// (un)set a hook which is ran before every call. It returns true if the command is done. +func (s *Server) SetPreHook(h Hook) { + s.mu.Lock() + s.preHook = h + s.mu.Unlock() +} + +func (s *Server) serve(l net.Listener) { + for { + conn, err := l.Accept() + if err != nil { + return + } + s.ServeConn(conn) + } +} + +// ServeConn handles a net.Conn. Nice with net.Pipe() +func (s *Server) ServeConn(conn net.Conn) { + s.wg.Add(1) + s.mu.Lock() + s.peers[conn] = struct{}{} + s.infoConns++ + s.mu.Unlock() + + go func() { + defer s.wg.Done() + defer conn.Close() + + s.servePeer(conn) + + s.mu.Lock() + delete(s.peers, conn) + s.mu.Unlock() + }() +} + +// Addr has the net.Addr struct +func (s *Server) Addr() *net.TCPAddr { + s.mu.Lock() + defer s.mu.Unlock() + if s.l == nil { + return nil + } + return s.l.Addr().(*net.TCPAddr) +} + +// Close a server started with NewServer. It will wait until all clients are +// closed. +func (s *Server) Close() { + s.mu.Lock() + if s.l != nil { + s.l.Close() + } + s.l = nil + s.mu.Unlock() + + s.wg.Wait() +} + +// Register a command. It can't have been registered before. Safe to call on a +// running server. +func (s *Server) Register(cmd string, f Cmd, options ...CmdOption) error { + s.mu.Lock() + defer s.mu.Unlock() + cmd = strings.ToUpper(cmd) + if _, ok := s.cmds[cmd]; ok { + return fmt.Errorf("command already registered: %s", cmd) + } + + meta := &cmdMeta{ + handler: f, + readOnly: false, + } + for _, option := range options { + option(meta) + } + s.cmds[cmd] = meta + + return nil +} + +func (s *Server) servePeer(c net.Conn) { + r := bufio.NewReader(c) + peer := &Peer{ + w: bufio.NewWriter(c), + } + + defer func() { + for _, f := range peer.onDisconnect { + f() + } + }() + + readCh := make(chan []string) + + go func() { + defer close(readCh) + + for { + args, err := readArray(r) + if err != nil { + peer.Close() + return + } + + readCh <- args + } + }() + + for args := range readCh { + s.Dispatch(peer, args) + peer.Flush() + + if peer.Closed() { + c.Close() + } + } +} + +func (s *Server) Dispatch(c *Peer, args []string) { + cmd, args := args[0], args[1:] + cmdUp := strings.ToUpper(cmd) + s.mu.Lock() + h := s.preHook + s.mu.Unlock() + if h != nil { + if h(c, cmdUp, args...) { + return + } + } + + s.mu.Lock() + cmdMeta, ok := s.cmds[cmdUp] + s.mu.Unlock() + if !ok { + c.WriteError(errUnknownCommand(cmd, args)) + return + } + + s.mu.Lock() + s.infoCmds++ + s.mu.Unlock() + cmdMeta.handler(c, cmdUp, args) + if c.SwitchResp3 != nil { + c.Resp3 = *c.SwitchResp3 + c.SwitchResp3 = nil + } +} + +// TotalCommands is total (known) commands since this the server started +func (s *Server) TotalCommands() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.infoCmds +} + +// IsRegisteredCommand checks if a command is registered +func (s *Server) IsRegisteredCommand(cmd string) bool { + s.mu.Lock() + defer s.mu.Unlock() + cmdUp := strings.ToUpper(cmd) + _, ok := s.cmds[cmdUp] + return ok +} + +// IsReadOnlyCommand checks if a command is marked as read-only +func (s *Server) IsReadOnlyCommand(cmd string) bool { + s.mu.Lock() + defer s.mu.Unlock() + cmdUp := strings.ToUpper(cmd) + if cmdMeta, ok := s.cmds[cmdUp]; ok { + return cmdMeta.readOnly + } + return false +} + +// ClientsLen gives the number of connected clients right now +func (s *Server) ClientsLen() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.peers) +} + +// TotalConnections give the number of clients connected since the server +// started, including the currently connected ones +func (s *Server) TotalConnections() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.infoConns +} + +// Peer is a client connected to the server +type Peer struct { + w *bufio.Writer + closed bool + Resp3 bool + SwitchResp3 *bool // we'll switch to this version _after_ the command + Ctx interface{} // anything goes, server won't touch this + onDisconnect []func() // list of callbacks + mu sync.Mutex // for Block() + ClientName string // client name set by CLIENT SETNAME +} + +func NewPeer(w *bufio.Writer) *Peer { + return &Peer{ + w: w, + } +} + +// Flush the write buffer. Called automatically after every redis command +func (c *Peer) Flush() { + c.mu.Lock() + defer c.mu.Unlock() + c.w.Flush() +} + +// Close the client connection after the current command is done. +func (c *Peer) Close() { + c.mu.Lock() + defer c.mu.Unlock() + c.closed = true +} + +// Return true if the peer connection closed. +func (c *Peer) Closed() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.closed +} + +// Register a function to execute on disconnect. There can be multiple +// functions registered. +func (c *Peer) OnDisconnect(f func()) { + c.onDisconnect = append(c.onDisconnect, f) +} + +// issue multiple calls, guarded with a mutex +func (c *Peer) Block(f func(*Writer)) { + c.mu.Lock() + defer c.mu.Unlock() + f(&Writer{c.w, c.Resp3}) +} + +// WriteError writes a redis 'Error' +func (c *Peer) WriteError(e string) { + c.Block(func(w *Writer) { + w.WriteError(e) + }) +} + +// WriteInline writes a redis inline string +func (c *Peer) WriteInline(s string) { + c.Block(func(w *Writer) { + w.WriteInline(s) + }) +} + +// WriteOK write the inline string `OK` +func (c *Peer) WriteOK() { + c.WriteInline("OK") +} + +// WriteBulk writes a bulk string +func (c *Peer) WriteBulk(s string) { + c.Block(func(w *Writer) { + w.WriteBulk(s) + }) +} + +// WriteNull writes a redis Null element +func (c *Peer) WriteNull() { + c.Block(func(w *Writer) { + w.WriteNull() + }) +} + +// WriteLen starts an array with the given length +func (c *Peer) WriteLen(n int) { + c.Block(func(w *Writer) { + w.WriteLen(n) + }) +} + +// WriteMapLen starts a map with the given length (number of keys) +func (c *Peer) WriteMapLen(n int) { + c.Block(func(w *Writer) { + w.WriteMapLen(n) + }) +} + +// WriteSetLen starts a set with the given length (number of elements) +func (c *Peer) WriteSetLen(n int) { + c.Block(func(w *Writer) { + w.WriteSetLen(n) + }) +} + +// WritePushLen starts a push-data array with the given length +func (c *Peer) WritePushLen(n int) { + c.Block(func(w *Writer) { + w.WritePushLen(n) + }) +} + +// WriteInt writes an integer +func (c *Peer) WriteInt(n int) { + c.Block(func(w *Writer) { + w.WriteInt(n) + }) +} + +// WriteFloat writes a float +func (c *Peer) WriteFloat(n float64) { + c.Block(func(w *Writer) { + w.WriteFloat(n) + }) +} + +// WriteRaw writes a raw redis response +func (c *Peer) WriteRaw(s string) { + c.Block(func(w *Writer) { + w.WriteRaw(s) + }) +} + +// WriteStrings is a helper to (bulk)write a string list +func (c *Peer) WriteStrings(strs []string) { + c.Block(func(w *Writer) { + w.WriteStrings(strs) + }) +} + +func toInline(s string) string { + return strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return ' ' + } + return r + }, s) +} + +// A Writer is given to the callback in Block() +type Writer struct { + w *bufio.Writer + resp3 bool +} + +// WriteError writes a redis 'Error' +func (w *Writer) WriteError(e string) { + fmt.Fprintf(w.w, "-%s\r\n", toInline(e)) +} + +func (w *Writer) WriteLen(n int) { + fmt.Fprintf(w.w, "*%d\r\n", n) +} + +func (w *Writer) WriteMapLen(n int) { + if w.resp3 { + fmt.Fprintf(w.w, "%%%d\r\n", n) + return + } + w.WriteLen(n * 2) +} + +func (w *Writer) WriteSetLen(n int) { + if w.resp3 { + fmt.Fprintf(w.w, "~%d\r\n", n) + return + } + w.WriteLen(n) +} + +func (w *Writer) WritePushLen(n int) { + if w.resp3 { + fmt.Fprintf(w.w, ">%d\r\n", n) + return + } + w.WriteLen(n) +} + +// WriteBulk writes a bulk string +func (w *Writer) WriteBulk(s string) { + fmt.Fprintf(w.w, "$%d\r\n%s\r\n", len(s), s) +} + +// WriteStrings writes a list of strings (bulk) +func (w *Writer) WriteStrings(strs []string) { + w.WriteLen(len(strs)) + for _, s := range strs { + w.WriteBulk(s) + } +} + +// WriteInt writes an integer +func (w *Writer) WriteInt(n int) { + fmt.Fprintf(w.w, ":%d\r\n", n) +} + +// WriteFloat writes a float +func (w *Writer) WriteFloat(n float64) { + if w.resp3 { + fmt.Fprintf(w.w, ",%s\r\n", formatFloat(n)) + return + } + w.WriteBulk(formatFloat(n)) +} + +// WriteNull writes a redis Null element +func (w *Writer) WriteNull() { + if w.resp3 { + fmt.Fprint(w.w, "_\r\n") + return + } + fmt.Fprintf(w.w, "$-1\r\n") +} + +// WriteInline writes a redis inline string +func (w *Writer) WriteInline(s string) { + fmt.Fprintf(w.w, "+%s\r\n", toInline(s)) +} + +// WriteRaw writes a raw redis response +func (w *Writer) WriteRaw(s string) { + fmt.Fprint(w.w, s) +} + +func (w *Writer) Flush() { + w.w.Flush() +} + +// formatFloat formats a float the way redis does. +// Redis uses a method called "grisu2", which we ported from C. +func formatFloat(v float64) string { + return fpconv.Dtoa(v) +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/size/readme.md b/scraper-go/vendor/github.com/alicebob/miniredis/v2/size/readme.md new file mode 100644 index 00000000..89220e45 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/size/readme.md @@ -0,0 +1,2 @@ + +Credits to DmitriyVTitov on his package https://github.com/DmitriyVTitov/size diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/size/size.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/size/size.go new file mode 100644 index 00000000..43fee6e2 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/size/size.go @@ -0,0 +1,138 @@ +package size + +import ( + "reflect" + "unsafe" +) + +// Of returns the size of 'v' in bytes. +// If there is an error during calculation, Of returns -1. +func Of(v interface{}) int { + // Cache with every visited pointer so we don't count two pointers + // to the same memory twice. + cache := make(map[uintptr]bool) + return sizeOf(reflect.Indirect(reflect.ValueOf(v)), cache) +} + +// sizeOf returns the number of bytes the actual data represented by v occupies in memory. +// If there is an error, sizeOf returns -1. +func sizeOf(v reflect.Value, cache map[uintptr]bool) int { + switch v.Kind() { + + case reflect.Array: + sum := 0 + for i := 0; i < v.Len(); i++ { + s := sizeOf(v.Index(i), cache) + if s < 0 { + return -1 + } + sum += s + } + + return sum + (v.Cap()-v.Len())*int(v.Type().Elem().Size()) + + case reflect.Slice: + // return 0 if this node has been visited already + if cache[v.Pointer()] { + return 0 + } + cache[v.Pointer()] = true + + sum := 0 + for i := 0; i < v.Len(); i++ { + s := sizeOf(v.Index(i), cache) + if s < 0 { + return -1 + } + sum += s + } + + sum += (v.Cap() - v.Len()) * int(v.Type().Elem().Size()) + + return sum + int(v.Type().Size()) + + case reflect.Struct: + sum := 0 + for i, n := 0, v.NumField(); i < n; i++ { + s := sizeOf(v.Field(i), cache) + if s < 0 { + return -1 + } + sum += s + } + + // Look for struct padding. + padding := int(v.Type().Size()) + for i, n := 0, v.NumField(); i < n; i++ { + padding -= int(v.Field(i).Type().Size()) + } + + return sum + padding + + case reflect.String: + s := v.String() + hdr := (*reflect.StringHeader)(unsafe.Pointer(&s)) + if cache[hdr.Data] { + return int(v.Type().Size()) + } + cache[hdr.Data] = true + return len(s) + int(v.Type().Size()) + + case reflect.Ptr: + // return Ptr size if this node has been visited already (infinite recursion) + if cache[v.Pointer()] { + return int(v.Type().Size()) + } + cache[v.Pointer()] = true + if v.IsNil() { + return int(reflect.New(v.Type()).Type().Size()) + } + s := sizeOf(reflect.Indirect(v), cache) + if s < 0 { + return -1 + } + return s + int(v.Type().Size()) + + case reflect.Bool, + reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Int, reflect.Uint, + reflect.Chan, + reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, + reflect.Func: + return int(v.Type().Size()) + + case reflect.Map: + // return 0 if this node has been visited already (infinite recursion) + if cache[v.Pointer()] { + return 0 + } + cache[v.Pointer()] = true + sum := 0 + keys := v.MapKeys() + for i := range keys { + val := v.MapIndex(keys[i]) + // calculate size of key and value separately + sv := sizeOf(val, cache) + if sv < 0 { + return -1 + } + sum += sv + sk := sizeOf(keys[i], cache) + if sk < 0 { + return -1 + } + sum += sk + } + // Include overhead due to unused map buckets. 10.79 comes + // from https://golang.org/src/runtime/map.go. + return sum + int(v.Type().Size()) + int(float64(len(keys))*10.79) + + case reflect.Interface: + return sizeOf(v.Elem(), cache) + int(v.Type().Size()) + + } + + return -1 +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/sorted_set.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/sorted_set.go new file mode 100644 index 00000000..96ebd5d7 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/sorted_set.go @@ -0,0 +1,98 @@ +package miniredis + +// The most KISS way to implement a sorted set. Luckily we don't care about +// performance that much. + +import ( + "sort" +) + +type direction int + +const ( + unsorted direction = iota + asc + desc +) + +type sortedSet map[string]float64 + +type ssElem struct { + score float64 + member string +} +type ssElems []ssElem + +type byScore ssElems + +func (sse byScore) Len() int { return len(sse) } +func (sse byScore) Swap(i, j int) { sse[i], sse[j] = sse[j], sse[i] } +func (sse byScore) Less(i, j int) bool { + if sse[i].score != sse[j].score { + return sse[i].score < sse[j].score + } + return sse[i].member < sse[j].member +} + +func newSortedSet() sortedSet { + return sortedSet{} +} + +func (ss *sortedSet) card() int { + return len(*ss) +} + +func (ss *sortedSet) set(score float64, member string) { + (*ss)[member] = score +} + +func (ss *sortedSet) get(member string) (float64, bool) { + v, ok := (*ss)[member] + return v, ok +} + +// elems gives the list of ssElem, ready to sort. +func (ss *sortedSet) elems() ssElems { + elems := make(ssElems, 0, len(*ss)) + for e, s := range *ss { + elems = append(elems, ssElem{s, e}) + } + return elems +} + +func (ss *sortedSet) byScore(d direction) ssElems { + elems := ss.elems() + sort.Sort(byScore(elems)) + if d == desc { + reverseElems(elems) + } + return ssElems(elems) +} + +// rankByScore gives the (0-based) index of member, or returns false. +func (ss *sortedSet) rankByScore(member string, d direction) (int, bool) { + if _, ok := (*ss)[member]; !ok { + return 0, false + } + for i, e := range ss.byScore(d) { + if e.member == member { + return i, true + } + } + // Can't happen + return 0, false +} + +func reverseSlice(o []string) { + for i := range make([]struct{}, len(o)/2) { + other := len(o) - 1 - i + o[i], o[other] = o[other], o[i] + } +} + +func reverseElems(o ssElems) { + for i := range make([]struct{}, len(o)/2) { + other := len(o) - 1 - i + o[i], o[other] = o[other], o[i] + } +} diff --git a/scraper-go/vendor/github.com/alicebob/miniredis/v2/stream.go b/scraper-go/vendor/github.com/alicebob/miniredis/v2/stream.go new file mode 100644 index 00000000..1f1fc9d7 --- /dev/null +++ b/scraper-go/vendor/github.com/alicebob/miniredis/v2/stream.go @@ -0,0 +1,515 @@ +// Basic stream implementation. + +package miniredis + +import ( + "errors" + "fmt" + "math" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// a Stream is a list of entries, lowest ID (oldest) first, and all "groups". +type streamKey struct { + entries []StreamEntry + groups map[string]*streamGroup + lastAllocatedID string + mu sync.Mutex +} + +// a StreamEntry is an entry in a stream. The ID is always of the form +// "123-123". +// Values is an ordered list of key-value pairs. +type StreamEntry struct { + ID string + Values []string +} + +type streamGroup struct { + stream *streamKey + lastID string + pending []pendingEntry + consumers map[string]*consumer +} + +type consumer struct { + numPendingEntries int + // these timestamps aren't tracked perfectly + lastSeen time.Time // "idle" XINFO key + lastSuccess time.Time // "inactive" XINFO key +} + +type pendingEntry struct { + id string + consumer string + deliveryCount int + lastDelivery time.Time +} + +func newStreamKey() *streamKey { + return &streamKey{ + groups: map[string]*streamGroup{}, + } +} + +// generateID doesn't lock the mutex +func (s *streamKey) generateID(ts uint64) string { + next := fmt.Sprintf("%d-%d", ts, 0) + if s.lastAllocatedID != "" && streamCmp(s.lastAllocatedID, next) >= 0 { + last, _ := parseStreamID(s.lastAllocatedID) + next = fmt.Sprintf("%d-%d", last[0], last[1]+1) + } + + lastID := s.lastIDUnlocked() + if streamCmp(lastID, next) >= 0 { + last, _ := parseStreamID(lastID) + next = fmt.Sprintf("%d-%d", last[0], last[1]+1) + } + + s.lastAllocatedID = next + return next +} + +// lastID locks the mutex +func (s *streamKey) lastID() string { + s.mu.Lock() + defer s.mu.Unlock() + + return s.lastIDUnlocked() +} + +// lastID doesn't lock the mutex +func (s *streamKey) lastIDUnlocked() string { + if len(s.entries) == 0 { + return "0-0" + } + + return s.entries[len(s.entries)-1].ID +} + +func (s *streamKey) copy() *streamKey { + s.mu.Lock() + defer s.mu.Unlock() + + cpy := &streamKey{ + entries: s.entries, + } + groups := map[string]*streamGroup{} + for k, v := range s.groups { + gr := v.copy() + gr.stream = cpy + groups[k] = gr + } + cpy.groups = groups + return cpy +} + +func parseStreamID(id string) ([2]uint64, error) { + var ( + res [2]uint64 + err error + ) + parts := strings.SplitN(id, "-", 2) + res[0], err = strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return res, errors.New(msgInvalidStreamID) + } + if len(parts) == 2 { + res[1], err = strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return res, errors.New(msgInvalidStreamID) + } + } + return res, nil +} + +// compares two stream IDs (of the full format: "123-123"). Returns: -1, 0, 1 +// The given IDs should be valid stream IDs. +func streamCmp(a, b string) int { + ap, _ := parseStreamID(a) + bp, _ := parseStreamID(b) + + switch { + case ap[0] < bp[0]: + return -1 + case ap[0] > bp[0]: + return 1 + case ap[1] < bp[1]: + return -1 + case ap[1] > bp[1]: + return 1 + default: + return 0 + } +} + +// formatStreamID makes a full id ("42-42") out of a partial one ("42") +func formatStreamID(id string) (string, error) { + var ts [2]uint64 + parts := strings.SplitN(id, "-", 2) + + if len(parts) > 0 { + p, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return "", errInvalidEntryID + } + ts[0] = p + } + if len(parts) > 1 { + p, err := strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return "", errInvalidEntryID + } + ts[1] = p + } + return fmt.Sprintf("%d-%d", ts[0], ts[1]), nil +} + +func formatStreamRangeBound(id string, start bool, reverse bool) (string, error) { + if id == "-" { + return "0-0", nil + } + + if id == "+" { + return fmt.Sprintf("%d-%d", uint64(math.MaxUint64), uint64(math.MaxUint64)), nil + } + + if id == "0" { + return "0-0", nil + } + + parts := strings.Split(id, "-") + if len(parts) == 2 { + return formatStreamID(id) + } + + // Incomplete IDs case + ts, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return "", errInvalidEntryID + } + + if (!start && !reverse) || (start && reverse) { + return fmt.Sprintf("%d-%d", ts, uint64(math.MaxUint64)), nil + } + + return fmt.Sprintf("%d-%d", ts, 0), nil +} + +func reversedStreamEntries(o []StreamEntry) []StreamEntry { + newStream := make([]StreamEntry, len(o)) + for i, e := range o { + newStream[len(o)-i-1] = e + } + return newStream +} + +func (s *streamKey) createGroup(group, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if _, ok := s.groups[group]; ok { + return errors.New("BUSYGROUP Consumer Group name already exists") + } + + if id == "$" { + id = s.lastIDUnlocked() + } + s.groups[group] = &streamGroup{ + stream: s, + lastID: id, + consumers: map[string]*consumer{}, + } + return nil +} + +// streamAdd adds an entry to a stream. Returns the new entry ID. +// If id is empty, "*", or "123-*", the ID will be generated automatically. +// `values` should have an even length. +func (s *streamKey) add(entryID string, values []string, now time.Time) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + + switch { + case entryID == "" || entryID == "*": + entryID = s.generateID(uint64(now.UnixMilli())) + default: + // "-*" + parts := strings.Split(entryID, "-") + if len(parts) == 2 && parts[1] == "*" { + if ts, err := strconv.ParseUint(parts[0], 10, 64); err == nil { + entryID = s.generateID(uint64(ts)) + } + } + } + + entryID, err := formatStreamID(entryID) + if err != nil { + return "", err + } + if entryID == "0-0" { + return "", errors.New(msgStreamIDZero) + } + if streamCmp(s.lastIDUnlocked(), entryID) != -1 { + return "", errors.New(msgStreamIDTooSmall) + } + + s.entries = append(s.entries, StreamEntry{ + ID: entryID, + Values: values, + }) + s.lastAllocatedID = entryID + return entryID, nil +} + +func (s *streamKey) trim(n int) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.entries) > n { + s.entries = s.entries[len(s.entries)-n:] + } +} + +// trimBefore deletes entries with an id less than the provided id +// and returns the number of entries deleted +func (s *streamKey) trimBefore(id string) int { + s.mu.Lock() + var delete []string + for _, entry := range s.entries { + if streamCmp(entry.ID, id) < 0 { + delete = append(delete, entry.ID) + } else { + break + } + } + s.mu.Unlock() + s.delete(delete) + return len(delete) +} + +// all entries after "id" +func (s *streamKey) after(id string) []StreamEntry { + s.mu.Lock() + defer s.mu.Unlock() + + pos := sort.Search(len(s.entries), func(i int) bool { + return streamCmp(id, s.entries[i].ID) < 0 + }) + return s.entries[pos:] +} + +// get a stream entry by ID +// Also returns the position in the entries slice, if found. +func (s *streamKey) get(id string) (int, *StreamEntry) { + s.mu.Lock() + defer s.mu.Unlock() + + pos := sort.Search(len(s.entries), func(i int) bool { + return streamCmp(id, s.entries[i].ID) <= 0 + }) + if len(s.entries) <= pos || s.entries[pos].ID != id { + return 0, nil + } + return pos, &s.entries[pos] +} + +func (g *streamGroup) readGroup( + now time.Time, + consumerID, + id string, + count int, + noack bool, +) []StreamEntry { + if id == ">" { + // undelivered messages + msgs := g.stream.after(g.lastID) + if len(msgs) == 0 { + return nil + } + + if count > 0 && len(msgs) > count { + msgs = msgs[:count] + } + + if !noack { + shouldAppend := len(g.pending) == 0 + for _, msg := range msgs { + if !shouldAppend { + shouldAppend = streamCmp(msg.ID, g.pending[len(g.pending)-1].id) == 1 + } + + var entry *pendingEntry + if shouldAppend { + g.pending = append(g.pending, pendingEntry{}) + entry = &g.pending[len(g.pending)-1] + } else { + var pos int + pos, entry = g.searchPending(msg.ID) + if entry == nil { + g.pending = append(g.pending[:pos+1], g.pending[pos:]...) + entry = &g.pending[pos] + } else { + g.consumers[entry.consumer].numPendingEntries-- + } + } + + *entry = pendingEntry{ + id: msg.ID, + consumer: consumerID, + deliveryCount: 1, + lastDelivery: now, + } + } + } + if _, ok := g.consumers[consumerID]; !ok { + g.consumers[consumerID] = &consumer{} + } + g.consumers[consumerID].numPendingEntries += len(msgs) + g.lastID = msgs[len(msgs)-1].ID + return msgs + } + + // re-deliver messages from the pending list. + // con := gr.consumers[consumerID] + msgs := g.pendingAfter(id) + var res []StreamEntry + for i, p := range msgs { + if p.consumer != consumerID { + continue + } + _, entry := g.stream.get(p.id) + // not found. Weird? + if entry == nil { + continue + } + p.deliveryCount += 1 + p.lastDelivery = now + msgs[i] = p + res = append(res, *entry) + } + return res +} + +func (g *streamGroup) searchPending(id string) (int, *pendingEntry) { + pos := sort.Search(len(g.pending), func(i int) bool { + return streamCmp(id, g.pending[i].id) <= 0 + }) + if pos >= len(g.pending) || g.pending[pos].id != id { + return pos, nil + } + return pos, &g.pending[pos] +} + +func (g *streamGroup) ack(ids []string) (int, error) { + count := 0 + for _, id := range ids { + if _, err := parseStreamID(id); err != nil { + return 0, errors.New(msgInvalidStreamID) + } + + pos, entry := g.searchPending(id) + if entry == nil { + continue + } + + consumer := g.consumers[entry.consumer] + consumer.numPendingEntries-- + + g.pending = append(g.pending[:pos], g.pending[pos+1:]...) + // don't count deleted entries + if _, e := g.stream.get(id); e == nil { + continue + } + count++ + } + return count, nil +} + +func (s *streamKey) delete(ids []string) (int, error) { + count := 0 + for _, id := range ids { + if _, err := parseStreamID(id); err != nil { + return 0, errors.New(msgInvalidStreamID) + } + + i, entry := s.get(id) + if entry == nil { + continue + } + + s.entries = append(s.entries[:i], s.entries[i+1:]...) + count++ + } + return count, nil +} + +func (g *streamGroup) pendingAfterOrEqual(id string) []pendingEntry { + pos := sort.Search(len(g.pending), func(i int) bool { + return streamCmp(id, g.pending[i].id) <= 0 + }) + return g.pending[pos:] +} + +func (g *streamGroup) pendingAfter(id string) []pendingEntry { + pos := sort.Search(len(g.pending), func(i int) bool { + return streamCmp(id, g.pending[i].id) < 0 + }) + return g.pending[pos:] +} + +func (g *streamGroup) pendingCount(consumer string) int { + n := 0 + for _, p := range g.activePending() { + if p.consumer == consumer { + n++ + } + } + return n +} + +// pending entries without the entries deleted from the group +func (g *streamGroup) activePending() []pendingEntry { + var pe []pendingEntry + for _, p := range g.pending { + // drop deleted ones + if _, e := g.stream.get(p.id); e == nil { + continue + } + p := p + pe = append(pe, p) + } + return pe +} + +func (g *streamGroup) copy() *streamGroup { + cns := map[string]*consumer{} + for k, v := range g.consumers { + c := *v + cns[k] = &c + } + return &streamGroup{ + // don't copy stream + lastID: g.lastID, + pending: g.pending, + consumers: cns, + } +} + +func (g *streamGroup) setLastSeen(c string, t time.Time) { + cons, ok := g.consumers[c] + if !ok { + cons = &consumer{} + } + cons.lastSeen = t + g.consumers[c] = cons +} + +func (g *streamGroup) setLastSuccess(c string, t time.Time) { + g.setLastSeen(c, t) + g.consumers[c].lastSuccess = t +} diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/LICENSE b/scraper-go/vendor/github.com/davecgh/go-spew/LICENSE new file mode 100644 index 00000000..bc52e96f --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/spew/bypass.go b/scraper-go/vendor/github.com/davecgh/go-spew/spew/bypass.go new file mode 100644 index 00000000..79299478 --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/spew/bypass.go @@ -0,0 +1,145 @@ +// Copyright (c) 2015-2016 Dave Collins +// +// Permission to use, copy, modify, and distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +// NOTE: Due to the following build constraints, this file will only be compiled +// when the code is not running on Google App Engine, compiled by GopherJS, and +// "-tags safe" is not added to the go build command line. The "disableunsafe" +// tag is deprecated and thus should not be used. +// Go versions prior to 1.4 are disabled because they use a different layout +// for interfaces which make the implementation of unsafeReflectValue more complex. +// +build !js,!appengine,!safe,!disableunsafe,go1.4 + +package spew + +import ( + "reflect" + "unsafe" +) + +const ( + // UnsafeDisabled is a build-time constant which specifies whether or + // not access to the unsafe package is available. + UnsafeDisabled = false + + // ptrSize is the size of a pointer on the current arch. + ptrSize = unsafe.Sizeof((*byte)(nil)) +) + +type flag uintptr + +var ( + // flagRO indicates whether the value field of a reflect.Value + // is read-only. + flagRO flag + + // flagAddr indicates whether the address of the reflect.Value's + // value may be taken. + flagAddr flag +) + +// flagKindMask holds the bits that make up the kind +// part of the flags field. In all the supported versions, +// it is in the lower 5 bits. +const flagKindMask = flag(0x1f) + +// Different versions of Go have used different +// bit layouts for the flags type. This table +// records the known combinations. +var okFlags = []struct { + ro, addr flag +}{{ + // From Go 1.4 to 1.5 + ro: 1 << 5, + addr: 1 << 7, +}, { + // Up to Go tip. + ro: 1<<5 | 1<<6, + addr: 1 << 8, +}} + +var flagValOffset = func() uintptr { + field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") + if !ok { + panic("reflect.Value has no flag field") + } + return field.Offset +}() + +// flagField returns a pointer to the flag field of a reflect.Value. +func flagField(v *reflect.Value) *flag { + return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset)) +} + +// unsafeReflectValue converts the passed reflect.Value into a one that bypasses +// the typical safety restrictions preventing access to unaddressable and +// unexported data. It works by digging the raw pointer to the underlying +// value out of the protected value and generating a new unprotected (unsafe) +// reflect.Value to it. +// +// This allows us to check for implementations of the Stringer and error +// interfaces to be used for pretty printing ordinarily unaddressable and +// inaccessible values such as unexported struct fields. +func unsafeReflectValue(v reflect.Value) reflect.Value { + if !v.IsValid() || (v.CanInterface() && v.CanAddr()) { + return v + } + flagFieldPtr := flagField(&v) + *flagFieldPtr &^= flagRO + *flagFieldPtr |= flagAddr + return v +} + +// Sanity checks against future reflect package changes +// to the type or semantics of the Value.flag field. +func init() { + field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") + if !ok { + panic("reflect.Value has no flag field") + } + if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() { + panic("reflect.Value flag field has changed kind") + } + type t0 int + var t struct { + A t0 + // t0 will have flagEmbedRO set. + t0 + // a will have flagStickyRO set + a t0 + } + vA := reflect.ValueOf(t).FieldByName("A") + va := reflect.ValueOf(t).FieldByName("a") + vt0 := reflect.ValueOf(t).FieldByName("t0") + + // Infer flagRO from the difference between the flags + // for the (otherwise identical) fields in t. + flagPublic := *flagField(&vA) + flagWithRO := *flagField(&va) | *flagField(&vt0) + flagRO = flagPublic ^ flagWithRO + + // Infer flagAddr from the difference between a value + // taken from a pointer and not. + vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A") + flagNoPtr := *flagField(&vA) + flagPtr := *flagField(&vPtrA) + flagAddr = flagNoPtr ^ flagPtr + + // Check that the inferred flags tally with one of the known versions. + for _, f := range okFlags { + if flagRO == f.ro && flagAddr == f.addr { + return + } + } + panic("reflect.Value read-only flag has changed semantics") +} diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go b/scraper-go/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go new file mode 100644 index 00000000..205c28d6 --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go @@ -0,0 +1,38 @@ +// Copyright (c) 2015-2016 Dave Collins +// +// Permission to use, copy, modify, and distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +// NOTE: Due to the following build constraints, this file will only be compiled +// when the code is running on Google App Engine, compiled by GopherJS, or +// "-tags safe" is added to the go build command line. The "disableunsafe" +// tag is deprecated and thus should not be used. +// +build js appengine safe disableunsafe !go1.4 + +package spew + +import "reflect" + +const ( + // UnsafeDisabled is a build-time constant which specifies whether or + // not access to the unsafe package is available. + UnsafeDisabled = true +) + +// unsafeReflectValue typically converts the passed reflect.Value into a one +// that bypasses the typical safety restrictions preventing access to +// unaddressable and unexported data. However, doing this relies on access to +// the unsafe package. This is a stub version which simply returns the passed +// reflect.Value when the unsafe package is not available. +func unsafeReflectValue(v reflect.Value) reflect.Value { + return v +} diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/spew/common.go b/scraper-go/vendor/github.com/davecgh/go-spew/spew/common.go new file mode 100644 index 00000000..1be8ce94 --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/spew/common.go @@ -0,0 +1,341 @@ +/* + * Copyright (c) 2013-2016 Dave Collins + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package spew + +import ( + "bytes" + "fmt" + "io" + "reflect" + "sort" + "strconv" +) + +// Some constants in the form of bytes to avoid string overhead. This mirrors +// the technique used in the fmt package. +var ( + panicBytes = []byte("(PANIC=") + plusBytes = []byte("+") + iBytes = []byte("i") + trueBytes = []byte("true") + falseBytes = []byte("false") + interfaceBytes = []byte("(interface {})") + commaNewlineBytes = []byte(",\n") + newlineBytes = []byte("\n") + openBraceBytes = []byte("{") + openBraceNewlineBytes = []byte("{\n") + closeBraceBytes = []byte("}") + asteriskBytes = []byte("*") + colonBytes = []byte(":") + colonSpaceBytes = []byte(": ") + openParenBytes = []byte("(") + closeParenBytes = []byte(")") + spaceBytes = []byte(" ") + pointerChainBytes = []byte("->") + nilAngleBytes = []byte("") + maxNewlineBytes = []byte("\n") + maxShortBytes = []byte("") + circularBytes = []byte("") + circularShortBytes = []byte("") + invalidAngleBytes = []byte("") + openBracketBytes = []byte("[") + closeBracketBytes = []byte("]") + percentBytes = []byte("%") + precisionBytes = []byte(".") + openAngleBytes = []byte("<") + closeAngleBytes = []byte(">") + openMapBytes = []byte("map[") + closeMapBytes = []byte("]") + lenEqualsBytes = []byte("len=") + capEqualsBytes = []byte("cap=") +) + +// hexDigits is used to map a decimal value to a hex digit. +var hexDigits = "0123456789abcdef" + +// catchPanic handles any panics that might occur during the handleMethods +// calls. +func catchPanic(w io.Writer, v reflect.Value) { + if err := recover(); err != nil { + w.Write(panicBytes) + fmt.Fprintf(w, "%v", err) + w.Write(closeParenBytes) + } +} + +// handleMethods attempts to call the Error and String methods on the underlying +// type the passed reflect.Value represents and outputes the result to Writer w. +// +// It handles panics in any called methods by catching and displaying the error +// as the formatted value. +func handleMethods(cs *ConfigState, w io.Writer, v reflect.Value) (handled bool) { + // We need an interface to check if the type implements the error or + // Stringer interface. However, the reflect package won't give us an + // interface on certain things like unexported struct fields in order + // to enforce visibility rules. We use unsafe, when it's available, + // to bypass these restrictions since this package does not mutate the + // values. + if !v.CanInterface() { + if UnsafeDisabled { + return false + } + + v = unsafeReflectValue(v) + } + + // Choose whether or not to do error and Stringer interface lookups against + // the base type or a pointer to the base type depending on settings. + // Technically calling one of these methods with a pointer receiver can + // mutate the value, however, types which choose to satisify an error or + // Stringer interface with a pointer receiver should not be mutating their + // state inside these interface methods. + if !cs.DisablePointerMethods && !UnsafeDisabled && !v.CanAddr() { + v = unsafeReflectValue(v) + } + if v.CanAddr() { + v = v.Addr() + } + + // Is it an error or Stringer? + switch iface := v.Interface().(type) { + case error: + defer catchPanic(w, v) + if cs.ContinueOnMethod { + w.Write(openParenBytes) + w.Write([]byte(iface.Error())) + w.Write(closeParenBytes) + w.Write(spaceBytes) + return false + } + + w.Write([]byte(iface.Error())) + return true + + case fmt.Stringer: + defer catchPanic(w, v) + if cs.ContinueOnMethod { + w.Write(openParenBytes) + w.Write([]byte(iface.String())) + w.Write(closeParenBytes) + w.Write(spaceBytes) + return false + } + w.Write([]byte(iface.String())) + return true + } + return false +} + +// printBool outputs a boolean value as true or false to Writer w. +func printBool(w io.Writer, val bool) { + if val { + w.Write(trueBytes) + } else { + w.Write(falseBytes) + } +} + +// printInt outputs a signed integer value to Writer w. +func printInt(w io.Writer, val int64, base int) { + w.Write([]byte(strconv.FormatInt(val, base))) +} + +// printUint outputs an unsigned integer value to Writer w. +func printUint(w io.Writer, val uint64, base int) { + w.Write([]byte(strconv.FormatUint(val, base))) +} + +// printFloat outputs a floating point value using the specified precision, +// which is expected to be 32 or 64bit, to Writer w. +func printFloat(w io.Writer, val float64, precision int) { + w.Write([]byte(strconv.FormatFloat(val, 'g', -1, precision))) +} + +// printComplex outputs a complex value using the specified float precision +// for the real and imaginary parts to Writer w. +func printComplex(w io.Writer, c complex128, floatPrecision int) { + r := real(c) + w.Write(openParenBytes) + w.Write([]byte(strconv.FormatFloat(r, 'g', -1, floatPrecision))) + i := imag(c) + if i >= 0 { + w.Write(plusBytes) + } + w.Write([]byte(strconv.FormatFloat(i, 'g', -1, floatPrecision))) + w.Write(iBytes) + w.Write(closeParenBytes) +} + +// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x' +// prefix to Writer w. +func printHexPtr(w io.Writer, p uintptr) { + // Null pointer. + num := uint64(p) + if num == 0 { + w.Write(nilAngleBytes) + return + } + + // Max uint64 is 16 bytes in hex + 2 bytes for '0x' prefix + buf := make([]byte, 18) + + // It's simpler to construct the hex string right to left. + base := uint64(16) + i := len(buf) - 1 + for num >= base { + buf[i] = hexDigits[num%base] + num /= base + i-- + } + buf[i] = hexDigits[num] + + // Add '0x' prefix. + i-- + buf[i] = 'x' + i-- + buf[i] = '0' + + // Strip unused leading bytes. + buf = buf[i:] + w.Write(buf) +} + +// valuesSorter implements sort.Interface to allow a slice of reflect.Value +// elements to be sorted. +type valuesSorter struct { + values []reflect.Value + strings []string // either nil or same len and values + cs *ConfigState +} + +// newValuesSorter initializes a valuesSorter instance, which holds a set of +// surrogate keys on which the data should be sorted. It uses flags in +// ConfigState to decide if and how to populate those surrogate keys. +func newValuesSorter(values []reflect.Value, cs *ConfigState) sort.Interface { + vs := &valuesSorter{values: values, cs: cs} + if canSortSimply(vs.values[0].Kind()) { + return vs + } + if !cs.DisableMethods { + vs.strings = make([]string, len(values)) + for i := range vs.values { + b := bytes.Buffer{} + if !handleMethods(cs, &b, vs.values[i]) { + vs.strings = nil + break + } + vs.strings[i] = b.String() + } + } + if vs.strings == nil && cs.SpewKeys { + vs.strings = make([]string, len(values)) + for i := range vs.values { + vs.strings[i] = Sprintf("%#v", vs.values[i].Interface()) + } + } + return vs +} + +// canSortSimply tests whether a reflect.Kind is a primitive that can be sorted +// directly, or whether it should be considered for sorting by surrogate keys +// (if the ConfigState allows it). +func canSortSimply(kind reflect.Kind) bool { + // This switch parallels valueSortLess, except for the default case. + switch kind { + case reflect.Bool: + return true + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + return true + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: + return true + case reflect.Float32, reflect.Float64: + return true + case reflect.String: + return true + case reflect.Uintptr: + return true + case reflect.Array: + return true + } + return false +} + +// Len returns the number of values in the slice. It is part of the +// sort.Interface implementation. +func (s *valuesSorter) Len() int { + return len(s.values) +} + +// Swap swaps the values at the passed indices. It is part of the +// sort.Interface implementation. +func (s *valuesSorter) Swap(i, j int) { + s.values[i], s.values[j] = s.values[j], s.values[i] + if s.strings != nil { + s.strings[i], s.strings[j] = s.strings[j], s.strings[i] + } +} + +// valueSortLess returns whether the first value should sort before the second +// value. It is used by valueSorter.Less as part of the sort.Interface +// implementation. +func valueSortLess(a, b reflect.Value) bool { + switch a.Kind() { + case reflect.Bool: + return !a.Bool() && b.Bool() + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + return a.Int() < b.Int() + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: + return a.Uint() < b.Uint() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.String: + return a.String() < b.String() + case reflect.Uintptr: + return a.Uint() < b.Uint() + case reflect.Array: + // Compare the contents of both arrays. + l := a.Len() + for i := 0; i < l; i++ { + av := a.Index(i) + bv := b.Index(i) + if av.Interface() == bv.Interface() { + continue + } + return valueSortLess(av, bv) + } + } + return a.String() < b.String() +} + +// Less returns whether the value at index i should sort before the +// value at index j. It is part of the sort.Interface implementation. +func (s *valuesSorter) Less(i, j int) bool { + if s.strings == nil { + return valueSortLess(s.values[i], s.values[j]) + } + return s.strings[i] < s.strings[j] +} + +// sortValues is a sort function that handles both native types and any type that +// can be converted to error or Stringer. Other inputs are sorted according to +// their Value.String() value to ensure display stability. +func sortValues(values []reflect.Value, cs *ConfigState) { + if len(values) == 0 { + return + } + sort.Sort(newValuesSorter(values, cs)) +} diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/spew/config.go b/scraper-go/vendor/github.com/davecgh/go-spew/spew/config.go new file mode 100644 index 00000000..2e3d22f3 --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/spew/config.go @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2013-2016 Dave Collins + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package spew + +import ( + "bytes" + "fmt" + "io" + "os" +) + +// ConfigState houses the configuration options used by spew to format and +// display values. There is a global instance, Config, that is used to control +// all top-level Formatter and Dump functionality. Each ConfigState instance +// provides methods equivalent to the top-level functions. +// +// The zero value for ConfigState provides no indentation. You would typically +// want to set it to a space or a tab. +// +// Alternatively, you can use NewDefaultConfig to get a ConfigState instance +// with default settings. See the documentation of NewDefaultConfig for default +// values. +type ConfigState struct { + // Indent specifies the string to use for each indentation level. The + // global config instance that all top-level functions use set this to a + // single space by default. If you would like more indentation, you might + // set this to a tab with "\t" or perhaps two spaces with " ". + Indent string + + // MaxDepth controls the maximum number of levels to descend into nested + // data structures. The default, 0, means there is no limit. + // + // NOTE: Circular data structures are properly detected, so it is not + // necessary to set this value unless you specifically want to limit deeply + // nested data structures. + MaxDepth int + + // DisableMethods specifies whether or not error and Stringer interfaces are + // invoked for types that implement them. + DisableMethods bool + + // DisablePointerMethods specifies whether or not to check for and invoke + // error and Stringer interfaces on types which only accept a pointer + // receiver when the current type is not a pointer. + // + // NOTE: This might be an unsafe action since calling one of these methods + // with a pointer receiver could technically mutate the value, however, + // in practice, types which choose to satisify an error or Stringer + // interface with a pointer receiver should not be mutating their state + // inside these interface methods. As a result, this option relies on + // access to the unsafe package, so it will not have any effect when + // running in environments without access to the unsafe package such as + // Google App Engine or with the "safe" build tag specified. + DisablePointerMethods bool + + // DisablePointerAddresses specifies whether to disable the printing of + // pointer addresses. This is useful when diffing data structures in tests. + DisablePointerAddresses bool + + // DisableCapacities specifies whether to disable the printing of capacities + // for arrays, slices, maps and channels. This is useful when diffing + // data structures in tests. + DisableCapacities bool + + // ContinueOnMethod specifies whether or not recursion should continue once + // a custom error or Stringer interface is invoked. The default, false, + // means it will print the results of invoking the custom error or Stringer + // interface and return immediately instead of continuing to recurse into + // the internals of the data type. + // + // NOTE: This flag does not have any effect if method invocation is disabled + // via the DisableMethods or DisablePointerMethods options. + ContinueOnMethod bool + + // SortKeys specifies map keys should be sorted before being printed. Use + // this to have a more deterministic, diffable output. Note that only + // native types (bool, int, uint, floats, uintptr and string) and types + // that support the error or Stringer interfaces (if methods are + // enabled) are supported, with other types sorted according to the + // reflect.Value.String() output which guarantees display stability. + SortKeys bool + + // SpewKeys specifies that, as a last resort attempt, map keys should + // be spewed to strings and sorted by those strings. This is only + // considered if SortKeys is true. + SpewKeys bool +} + +// Config is the active configuration of the top-level functions. +// The configuration can be changed by modifying the contents of spew.Config. +var Config = ConfigState{Indent: " "} + +// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were +// passed with a Formatter interface returned by c.NewFormatter. It returns +// the formatted string as a value that satisfies error. See NewFormatter +// for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Errorf(format, c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Errorf(format string, a ...interface{}) (err error) { + return fmt.Errorf(format, c.convertArgs(a)...) +} + +// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were +// passed with a Formatter interface returned by c.NewFormatter. It returns +// the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Fprint(w, c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Fprint(w io.Writer, a ...interface{}) (n int, err error) { + return fmt.Fprint(w, c.convertArgs(a)...) +} + +// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were +// passed with a Formatter interface returned by c.NewFormatter. It returns +// the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Fprintf(w, format, c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(w, format, c.convertArgs(a)...) +} + +// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it +// passed with a Formatter interface returned by c.NewFormatter. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Fprintln(w, c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Fprintln(w io.Writer, a ...interface{}) (n int, err error) { + return fmt.Fprintln(w, c.convertArgs(a)...) +} + +// Print is a wrapper for fmt.Print that treats each argument as if it were +// passed with a Formatter interface returned by c.NewFormatter. It returns +// the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Print(c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Print(a ...interface{}) (n int, err error) { + return fmt.Print(c.convertArgs(a)...) +} + +// Printf is a wrapper for fmt.Printf that treats each argument as if it were +// passed with a Formatter interface returned by c.NewFormatter. It returns +// the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Printf(format, c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Printf(format, c.convertArgs(a)...) +} + +// Println is a wrapper for fmt.Println that treats each argument as if it were +// passed with a Formatter interface returned by c.NewFormatter. It returns +// the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Println(c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Println(a ...interface{}) (n int, err error) { + return fmt.Println(c.convertArgs(a)...) +} + +// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were +// passed with a Formatter interface returned by c.NewFormatter. It returns +// the resulting string. See NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Sprint(c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Sprint(a ...interface{}) string { + return fmt.Sprint(c.convertArgs(a)...) +} + +// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were +// passed with a Formatter interface returned by c.NewFormatter. It returns +// the resulting string. See NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Sprintf(format, c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Sprintf(format string, a ...interface{}) string { + return fmt.Sprintf(format, c.convertArgs(a)...) +} + +// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it +// were passed with a Formatter interface returned by c.NewFormatter. It +// returns the resulting string. See NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Sprintln(c.NewFormatter(a), c.NewFormatter(b)) +func (c *ConfigState) Sprintln(a ...interface{}) string { + return fmt.Sprintln(c.convertArgs(a)...) +} + +/* +NewFormatter returns a custom formatter that satisfies the fmt.Formatter +interface. As a result, it integrates cleanly with standard fmt package +printing functions. The formatter is useful for inline printing of smaller data +types similar to the standard %v format specifier. + +The custom formatter only responds to the %v (most compact), %+v (adds pointer +addresses), %#v (adds types), and %#+v (adds types and pointer addresses) verb +combinations. Any other verbs such as %x and %q will be sent to the the +standard fmt package for formatting. In addition, the custom formatter ignores +the width and precision arguments (however they will still work on the format +specifiers not handled by the custom formatter). + +Typically this function shouldn't be called directly. It is much easier to make +use of the custom formatter by calling one of the convenience functions such as +c.Printf, c.Println, or c.Printf. +*/ +func (c *ConfigState) NewFormatter(v interface{}) fmt.Formatter { + return newFormatter(c, v) +} + +// Fdump formats and displays the passed arguments to io.Writer w. It formats +// exactly the same as Dump. +func (c *ConfigState) Fdump(w io.Writer, a ...interface{}) { + fdump(c, w, a...) +} + +/* +Dump displays the passed parameters to standard out with newlines, customizable +indentation, and additional debug information such as complete types and all +pointer addresses used to indirect to the final value. It provides the +following features over the built-in printing facilities provided by the fmt +package: + + * Pointers are dereferenced and followed + * Circular data structures are detected and handled properly + * Custom Stringer/error interfaces are optionally invoked, including + on unexported types + * Custom types which only implement the Stringer/error interfaces via + a pointer receiver are optionally invoked when passing non-pointer + variables + * Byte arrays and slices are dumped like the hexdump -C command which + includes offsets, byte values in hex, and ASCII output + +The configuration options are controlled by modifying the public members +of c. See ConfigState for options documentation. + +See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to +get the formatted result as a string. +*/ +func (c *ConfigState) Dump(a ...interface{}) { + fdump(c, os.Stdout, a...) +} + +// Sdump returns a string with the passed arguments formatted exactly the same +// as Dump. +func (c *ConfigState) Sdump(a ...interface{}) string { + var buf bytes.Buffer + fdump(c, &buf, a...) + return buf.String() +} + +// convertArgs accepts a slice of arguments and returns a slice of the same +// length with each argument converted to a spew Formatter interface using +// the ConfigState associated with s. +func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{}) { + formatters = make([]interface{}, len(args)) + for index, arg := range args { + formatters[index] = newFormatter(c, arg) + } + return formatters +} + +// NewDefaultConfig returns a ConfigState with the following default settings. +// +// Indent: " " +// MaxDepth: 0 +// DisableMethods: false +// DisablePointerMethods: false +// ContinueOnMethod: false +// SortKeys: false +func NewDefaultConfig() *ConfigState { + return &ConfigState{Indent: " "} +} diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/spew/doc.go b/scraper-go/vendor/github.com/davecgh/go-spew/spew/doc.go new file mode 100644 index 00000000..aacaac6f --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/spew/doc.go @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2013-2016 Dave Collins + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* +Package spew implements a deep pretty printer for Go data structures to aid in +debugging. + +A quick overview of the additional features spew provides over the built-in +printing facilities for Go data types are as follows: + + * Pointers are dereferenced and followed + * Circular data structures are detected and handled properly + * Custom Stringer/error interfaces are optionally invoked, including + on unexported types + * Custom types which only implement the Stringer/error interfaces via + a pointer receiver are optionally invoked when passing non-pointer + variables + * Byte arrays and slices are dumped like the hexdump -C command which + includes offsets, byte values in hex, and ASCII output (only when using + Dump style) + +There are two different approaches spew allows for dumping Go data structures: + + * Dump style which prints with newlines, customizable indentation, + and additional debug information such as types and all pointer addresses + used to indirect to the final value + * A custom Formatter interface that integrates cleanly with the standard fmt + package and replaces %v, %+v, %#v, and %#+v to provide inline printing + similar to the default %v while providing the additional functionality + outlined above and passing unsupported format verbs such as %x and %q + along to fmt + +Quick Start + +This section demonstrates how to quickly get started with spew. See the +sections below for further details on formatting and configuration options. + +To dump a variable with full newlines, indentation, type, and pointer +information use Dump, Fdump, or Sdump: + spew.Dump(myVar1, myVar2, ...) + spew.Fdump(someWriter, myVar1, myVar2, ...) + str := spew.Sdump(myVar1, myVar2, ...) + +Alternatively, if you would prefer to use format strings with a compacted inline +printing style, use the convenience wrappers Printf, Fprintf, etc with +%v (most compact), %+v (adds pointer addresses), %#v (adds types), or +%#+v (adds types and pointer addresses): + spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) + spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) + spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) + spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) + +Configuration Options + +Configuration of spew is handled by fields in the ConfigState type. For +convenience, all of the top-level functions use a global state available +via the spew.Config global. + +It is also possible to create a ConfigState instance that provides methods +equivalent to the top-level functions. This allows concurrent configuration +options. See the ConfigState documentation for more details. + +The following configuration options are available: + * Indent + String to use for each indentation level for Dump functions. + It is a single space by default. A popular alternative is "\t". + + * MaxDepth + Maximum number of levels to descend into nested data structures. + There is no limit by default. + + * DisableMethods + Disables invocation of error and Stringer interface methods. + Method invocation is enabled by default. + + * DisablePointerMethods + Disables invocation of error and Stringer interface methods on types + which only accept pointer receivers from non-pointer variables. + Pointer method invocation is enabled by default. + + * DisablePointerAddresses + DisablePointerAddresses specifies whether to disable the printing of + pointer addresses. This is useful when diffing data structures in tests. + + * DisableCapacities + DisableCapacities specifies whether to disable the printing of + capacities for arrays, slices, maps and channels. This is useful when + diffing data structures in tests. + + * ContinueOnMethod + Enables recursion into types after invoking error and Stringer interface + methods. Recursion after method invocation is disabled by default. + + * SortKeys + Specifies map keys should be sorted before being printed. Use + this to have a more deterministic, diffable output. Note that + only native types (bool, int, uint, floats, uintptr and string) + and types which implement error or Stringer interfaces are + supported with other types sorted according to the + reflect.Value.String() output which guarantees display + stability. Natural map order is used by default. + + * SpewKeys + Specifies that, as a last resort attempt, map keys should be + spewed to strings and sorted by those strings. This is only + considered if SortKeys is true. + +Dump Usage + +Simply call spew.Dump with a list of variables you want to dump: + + spew.Dump(myVar1, myVar2, ...) + +You may also call spew.Fdump if you would prefer to output to an arbitrary +io.Writer. For example, to dump to standard error: + + spew.Fdump(os.Stderr, myVar1, myVar2, ...) + +A third option is to call spew.Sdump to get the formatted output as a string: + + str := spew.Sdump(myVar1, myVar2, ...) + +Sample Dump Output + +See the Dump example for details on the setup of the types and variables being +shown here. + + (main.Foo) { + unexportedField: (*main.Bar)(0xf84002e210)({ + flag: (main.Flag) flagTwo, + data: (uintptr) + }), + ExportedField: (map[interface {}]interface {}) (len=1) { + (string) (len=3) "one": (bool) true + } + } + +Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C +command as shown. + ([]uint8) (len=32 cap=32) { + 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... | + 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0| + 00000020 31 32 |12| + } + +Custom Formatter + +Spew provides a custom formatter that implements the fmt.Formatter interface +so that it integrates cleanly with standard fmt package printing functions. The +formatter is useful for inline printing of smaller data types similar to the +standard %v format specifier. + +The custom formatter only responds to the %v (most compact), %+v (adds pointer +addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb +combinations. Any other verbs such as %x and %q will be sent to the the +standard fmt package for formatting. In addition, the custom formatter ignores +the width and precision arguments (however they will still work on the format +specifiers not handled by the custom formatter). + +Custom Formatter Usage + +The simplest way to make use of the spew custom formatter is to call one of the +convenience functions such as spew.Printf, spew.Println, or spew.Printf. The +functions have syntax you are most likely already familiar with: + + spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) + spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) + spew.Println(myVar, myVar2) + spew.Fprintf(os.Stderr, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) + spew.Fprintf(os.Stderr, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) + +See the Index for the full list convenience functions. + +Sample Formatter Output + +Double pointer to a uint8: + %v: <**>5 + %+v: <**>(0xf8400420d0->0xf8400420c8)5 + %#v: (**uint8)5 + %#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5 + +Pointer to circular struct with a uint8 field and a pointer to itself: + %v: <*>{1 <*>} + %+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)} + %#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)} + %#+v: (*main.circular)(0xf84003e260){ui8:(uint8)1 c:(*main.circular)(0xf84003e260)} + +See the Printf example for details on the setup of variables being shown +here. + +Errors + +Since it is possible for custom Stringer/error interfaces to panic, spew +detects them and handles them internally by printing the panic information +inline with the output. Since spew is intended to provide deep pretty printing +capabilities on structures, it intentionally does not return any errors. +*/ +package spew diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/spew/dump.go b/scraper-go/vendor/github.com/davecgh/go-spew/spew/dump.go new file mode 100644 index 00000000..f78d89fc --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/spew/dump.go @@ -0,0 +1,509 @@ +/* + * Copyright (c) 2013-2016 Dave Collins + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package spew + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "os" + "reflect" + "regexp" + "strconv" + "strings" +) + +var ( + // uint8Type is a reflect.Type representing a uint8. It is used to + // convert cgo types to uint8 slices for hexdumping. + uint8Type = reflect.TypeOf(uint8(0)) + + // cCharRE is a regular expression that matches a cgo char. + // It is used to detect character arrays to hexdump them. + cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`) + + // cUnsignedCharRE is a regular expression that matches a cgo unsigned + // char. It is used to detect unsigned character arrays to hexdump + // them. + cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`) + + // cUint8tCharRE is a regular expression that matches a cgo uint8_t. + // It is used to detect uint8_t arrays to hexdump them. + cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`) +) + +// dumpState contains information about the state of a dump operation. +type dumpState struct { + w io.Writer + depth int + pointers map[uintptr]int + ignoreNextType bool + ignoreNextIndent bool + cs *ConfigState +} + +// indent performs indentation according to the depth level and cs.Indent +// option. +func (d *dumpState) indent() { + if d.ignoreNextIndent { + d.ignoreNextIndent = false + return + } + d.w.Write(bytes.Repeat([]byte(d.cs.Indent), d.depth)) +} + +// unpackValue returns values inside of non-nil interfaces when possible. +// This is useful for data types like structs, arrays, slices, and maps which +// can contain varying types packed inside an interface. +func (d *dumpState) unpackValue(v reflect.Value) reflect.Value { + if v.Kind() == reflect.Interface && !v.IsNil() { + v = v.Elem() + } + return v +} + +// dumpPtr handles formatting of pointers by indirecting them as necessary. +func (d *dumpState) dumpPtr(v reflect.Value) { + // Remove pointers at or below the current depth from map used to detect + // circular refs. + for k, depth := range d.pointers { + if depth >= d.depth { + delete(d.pointers, k) + } + } + + // Keep list of all dereferenced pointers to show later. + pointerChain := make([]uintptr, 0) + + // Figure out how many levels of indirection there are by dereferencing + // pointers and unpacking interfaces down the chain while detecting circular + // references. + nilFound := false + cycleFound := false + indirects := 0 + ve := v + for ve.Kind() == reflect.Ptr { + if ve.IsNil() { + nilFound = true + break + } + indirects++ + addr := ve.Pointer() + pointerChain = append(pointerChain, addr) + if pd, ok := d.pointers[addr]; ok && pd < d.depth { + cycleFound = true + indirects-- + break + } + d.pointers[addr] = d.depth + + ve = ve.Elem() + if ve.Kind() == reflect.Interface { + if ve.IsNil() { + nilFound = true + break + } + ve = ve.Elem() + } + } + + // Display type information. + d.w.Write(openParenBytes) + d.w.Write(bytes.Repeat(asteriskBytes, indirects)) + d.w.Write([]byte(ve.Type().String())) + d.w.Write(closeParenBytes) + + // Display pointer information. + if !d.cs.DisablePointerAddresses && len(pointerChain) > 0 { + d.w.Write(openParenBytes) + for i, addr := range pointerChain { + if i > 0 { + d.w.Write(pointerChainBytes) + } + printHexPtr(d.w, addr) + } + d.w.Write(closeParenBytes) + } + + // Display dereferenced value. + d.w.Write(openParenBytes) + switch { + case nilFound: + d.w.Write(nilAngleBytes) + + case cycleFound: + d.w.Write(circularBytes) + + default: + d.ignoreNextType = true + d.dump(ve) + } + d.w.Write(closeParenBytes) +} + +// dumpSlice handles formatting of arrays and slices. Byte (uint8 under +// reflection) arrays and slices are dumped in hexdump -C fashion. +func (d *dumpState) dumpSlice(v reflect.Value) { + // Determine whether this type should be hex dumped or not. Also, + // for types which should be hexdumped, try to use the underlying data + // first, then fall back to trying to convert them to a uint8 slice. + var buf []uint8 + doConvert := false + doHexDump := false + numEntries := v.Len() + if numEntries > 0 { + vt := v.Index(0).Type() + vts := vt.String() + switch { + // C types that need to be converted. + case cCharRE.MatchString(vts): + fallthrough + case cUnsignedCharRE.MatchString(vts): + fallthrough + case cUint8tCharRE.MatchString(vts): + doConvert = true + + // Try to use existing uint8 slices and fall back to converting + // and copying if that fails. + case vt.Kind() == reflect.Uint8: + // We need an addressable interface to convert the type + // to a byte slice. However, the reflect package won't + // give us an interface on certain things like + // unexported struct fields in order to enforce + // visibility rules. We use unsafe, when available, to + // bypass these restrictions since this package does not + // mutate the values. + vs := v + if !vs.CanInterface() || !vs.CanAddr() { + vs = unsafeReflectValue(vs) + } + if !UnsafeDisabled { + vs = vs.Slice(0, numEntries) + + // Use the existing uint8 slice if it can be + // type asserted. + iface := vs.Interface() + if slice, ok := iface.([]uint8); ok { + buf = slice + doHexDump = true + break + } + } + + // The underlying data needs to be converted if it can't + // be type asserted to a uint8 slice. + doConvert = true + } + + // Copy and convert the underlying type if needed. + if doConvert && vt.ConvertibleTo(uint8Type) { + // Convert and copy each element into a uint8 byte + // slice. + buf = make([]uint8, numEntries) + for i := 0; i < numEntries; i++ { + vv := v.Index(i) + buf[i] = uint8(vv.Convert(uint8Type).Uint()) + } + doHexDump = true + } + } + + // Hexdump the entire slice as needed. + if doHexDump { + indent := strings.Repeat(d.cs.Indent, d.depth) + str := indent + hex.Dump(buf) + str = strings.Replace(str, "\n", "\n"+indent, -1) + str = strings.TrimRight(str, d.cs.Indent) + d.w.Write([]byte(str)) + return + } + + // Recursively call dump for each item. + for i := 0; i < numEntries; i++ { + d.dump(d.unpackValue(v.Index(i))) + if i < (numEntries - 1) { + d.w.Write(commaNewlineBytes) + } else { + d.w.Write(newlineBytes) + } + } +} + +// dump is the main workhorse for dumping a value. It uses the passed reflect +// value to figure out what kind of object we are dealing with and formats it +// appropriately. It is a recursive function, however circular data structures +// are detected and handled properly. +func (d *dumpState) dump(v reflect.Value) { + // Handle invalid reflect values immediately. + kind := v.Kind() + if kind == reflect.Invalid { + d.w.Write(invalidAngleBytes) + return + } + + // Handle pointers specially. + if kind == reflect.Ptr { + d.indent() + d.dumpPtr(v) + return + } + + // Print type information unless already handled elsewhere. + if !d.ignoreNextType { + d.indent() + d.w.Write(openParenBytes) + d.w.Write([]byte(v.Type().String())) + d.w.Write(closeParenBytes) + d.w.Write(spaceBytes) + } + d.ignoreNextType = false + + // Display length and capacity if the built-in len and cap functions + // work with the value's kind and the len/cap itself is non-zero. + valueLen, valueCap := 0, 0 + switch v.Kind() { + case reflect.Array, reflect.Slice, reflect.Chan: + valueLen, valueCap = v.Len(), v.Cap() + case reflect.Map, reflect.String: + valueLen = v.Len() + } + if valueLen != 0 || !d.cs.DisableCapacities && valueCap != 0 { + d.w.Write(openParenBytes) + if valueLen != 0 { + d.w.Write(lenEqualsBytes) + printInt(d.w, int64(valueLen), 10) + } + if !d.cs.DisableCapacities && valueCap != 0 { + if valueLen != 0 { + d.w.Write(spaceBytes) + } + d.w.Write(capEqualsBytes) + printInt(d.w, int64(valueCap), 10) + } + d.w.Write(closeParenBytes) + d.w.Write(spaceBytes) + } + + // Call Stringer/error interfaces if they exist and the handle methods flag + // is enabled + if !d.cs.DisableMethods { + if (kind != reflect.Invalid) && (kind != reflect.Interface) { + if handled := handleMethods(d.cs, d.w, v); handled { + return + } + } + } + + switch kind { + case reflect.Invalid: + // Do nothing. We should never get here since invalid has already + // been handled above. + + case reflect.Bool: + printBool(d.w, v.Bool()) + + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + printInt(d.w, v.Int(), 10) + + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: + printUint(d.w, v.Uint(), 10) + + case reflect.Float32: + printFloat(d.w, v.Float(), 32) + + case reflect.Float64: + printFloat(d.w, v.Float(), 64) + + case reflect.Complex64: + printComplex(d.w, v.Complex(), 32) + + case reflect.Complex128: + printComplex(d.w, v.Complex(), 64) + + case reflect.Slice: + if v.IsNil() { + d.w.Write(nilAngleBytes) + break + } + fallthrough + + case reflect.Array: + d.w.Write(openBraceNewlineBytes) + d.depth++ + if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { + d.indent() + d.w.Write(maxNewlineBytes) + } else { + d.dumpSlice(v) + } + d.depth-- + d.indent() + d.w.Write(closeBraceBytes) + + case reflect.String: + d.w.Write([]byte(strconv.Quote(v.String()))) + + case reflect.Interface: + // The only time we should get here is for nil interfaces due to + // unpackValue calls. + if v.IsNil() { + d.w.Write(nilAngleBytes) + } + + case reflect.Ptr: + // Do nothing. We should never get here since pointers have already + // been handled above. + + case reflect.Map: + // nil maps should be indicated as different than empty maps + if v.IsNil() { + d.w.Write(nilAngleBytes) + break + } + + d.w.Write(openBraceNewlineBytes) + d.depth++ + if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { + d.indent() + d.w.Write(maxNewlineBytes) + } else { + numEntries := v.Len() + keys := v.MapKeys() + if d.cs.SortKeys { + sortValues(keys, d.cs) + } + for i, key := range keys { + d.dump(d.unpackValue(key)) + d.w.Write(colonSpaceBytes) + d.ignoreNextIndent = true + d.dump(d.unpackValue(v.MapIndex(key))) + if i < (numEntries - 1) { + d.w.Write(commaNewlineBytes) + } else { + d.w.Write(newlineBytes) + } + } + } + d.depth-- + d.indent() + d.w.Write(closeBraceBytes) + + case reflect.Struct: + d.w.Write(openBraceNewlineBytes) + d.depth++ + if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { + d.indent() + d.w.Write(maxNewlineBytes) + } else { + vt := v.Type() + numFields := v.NumField() + for i := 0; i < numFields; i++ { + d.indent() + vtf := vt.Field(i) + d.w.Write([]byte(vtf.Name)) + d.w.Write(colonSpaceBytes) + d.ignoreNextIndent = true + d.dump(d.unpackValue(v.Field(i))) + if i < (numFields - 1) { + d.w.Write(commaNewlineBytes) + } else { + d.w.Write(newlineBytes) + } + } + } + d.depth-- + d.indent() + d.w.Write(closeBraceBytes) + + case reflect.Uintptr: + printHexPtr(d.w, uintptr(v.Uint())) + + case reflect.UnsafePointer, reflect.Chan, reflect.Func: + printHexPtr(d.w, v.Pointer()) + + // There were not any other types at the time this code was written, but + // fall back to letting the default fmt package handle it in case any new + // types are added. + default: + if v.CanInterface() { + fmt.Fprintf(d.w, "%v", v.Interface()) + } else { + fmt.Fprintf(d.w, "%v", v.String()) + } + } +} + +// fdump is a helper function to consolidate the logic from the various public +// methods which take varying writers and config states. +func fdump(cs *ConfigState, w io.Writer, a ...interface{}) { + for _, arg := range a { + if arg == nil { + w.Write(interfaceBytes) + w.Write(spaceBytes) + w.Write(nilAngleBytes) + w.Write(newlineBytes) + continue + } + + d := dumpState{w: w, cs: cs} + d.pointers = make(map[uintptr]int) + d.dump(reflect.ValueOf(arg)) + d.w.Write(newlineBytes) + } +} + +// Fdump formats and displays the passed arguments to io.Writer w. It formats +// exactly the same as Dump. +func Fdump(w io.Writer, a ...interface{}) { + fdump(&Config, w, a...) +} + +// Sdump returns a string with the passed arguments formatted exactly the same +// as Dump. +func Sdump(a ...interface{}) string { + var buf bytes.Buffer + fdump(&Config, &buf, a...) + return buf.String() +} + +/* +Dump displays the passed parameters to standard out with newlines, customizable +indentation, and additional debug information such as complete types and all +pointer addresses used to indirect to the final value. It provides the +following features over the built-in printing facilities provided by the fmt +package: + + * Pointers are dereferenced and followed + * Circular data structures are detected and handled properly + * Custom Stringer/error interfaces are optionally invoked, including + on unexported types + * Custom types which only implement the Stringer/error interfaces via + a pointer receiver are optionally invoked when passing non-pointer + variables + * Byte arrays and slices are dumped like the hexdump -C command which + includes offsets, byte values in hex, and ASCII output + +The configuration options are controlled by an exported package global, +spew.Config. See ConfigState for options documentation. + +See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to +get the formatted result as a string. +*/ +func Dump(a ...interface{}) { + fdump(&Config, os.Stdout, a...) +} diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/spew/format.go b/scraper-go/vendor/github.com/davecgh/go-spew/spew/format.go new file mode 100644 index 00000000..b04edb7d --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/spew/format.go @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2013-2016 Dave Collins + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package spew + +import ( + "bytes" + "fmt" + "reflect" + "strconv" + "strings" +) + +// supportedFlags is a list of all the character flags supported by fmt package. +const supportedFlags = "0-+# " + +// formatState implements the fmt.Formatter interface and contains information +// about the state of a formatting operation. The NewFormatter function can +// be used to get a new Formatter which can be used directly as arguments +// in standard fmt package printing calls. +type formatState struct { + value interface{} + fs fmt.State + depth int + pointers map[uintptr]int + ignoreNextType bool + cs *ConfigState +} + +// buildDefaultFormat recreates the original format string without precision +// and width information to pass in to fmt.Sprintf in the case of an +// unrecognized type. Unless new types are added to the language, this +// function won't ever be called. +func (f *formatState) buildDefaultFormat() (format string) { + buf := bytes.NewBuffer(percentBytes) + + for _, flag := range supportedFlags { + if f.fs.Flag(int(flag)) { + buf.WriteRune(flag) + } + } + + buf.WriteRune('v') + + format = buf.String() + return format +} + +// constructOrigFormat recreates the original format string including precision +// and width information to pass along to the standard fmt package. This allows +// automatic deferral of all format strings this package doesn't support. +func (f *formatState) constructOrigFormat(verb rune) (format string) { + buf := bytes.NewBuffer(percentBytes) + + for _, flag := range supportedFlags { + if f.fs.Flag(int(flag)) { + buf.WriteRune(flag) + } + } + + if width, ok := f.fs.Width(); ok { + buf.WriteString(strconv.Itoa(width)) + } + + if precision, ok := f.fs.Precision(); ok { + buf.Write(precisionBytes) + buf.WriteString(strconv.Itoa(precision)) + } + + buf.WriteRune(verb) + + format = buf.String() + return format +} + +// unpackValue returns values inside of non-nil interfaces when possible and +// ensures that types for values which have been unpacked from an interface +// are displayed when the show types flag is also set. +// This is useful for data types like structs, arrays, slices, and maps which +// can contain varying types packed inside an interface. +func (f *formatState) unpackValue(v reflect.Value) reflect.Value { + if v.Kind() == reflect.Interface { + f.ignoreNextType = false + if !v.IsNil() { + v = v.Elem() + } + } + return v +} + +// formatPtr handles formatting of pointers by indirecting them as necessary. +func (f *formatState) formatPtr(v reflect.Value) { + // Display nil if top level pointer is nil. + showTypes := f.fs.Flag('#') + if v.IsNil() && (!showTypes || f.ignoreNextType) { + f.fs.Write(nilAngleBytes) + return + } + + // Remove pointers at or below the current depth from map used to detect + // circular refs. + for k, depth := range f.pointers { + if depth >= f.depth { + delete(f.pointers, k) + } + } + + // Keep list of all dereferenced pointers to possibly show later. + pointerChain := make([]uintptr, 0) + + // Figure out how many levels of indirection there are by derferencing + // pointers and unpacking interfaces down the chain while detecting circular + // references. + nilFound := false + cycleFound := false + indirects := 0 + ve := v + for ve.Kind() == reflect.Ptr { + if ve.IsNil() { + nilFound = true + break + } + indirects++ + addr := ve.Pointer() + pointerChain = append(pointerChain, addr) + if pd, ok := f.pointers[addr]; ok && pd < f.depth { + cycleFound = true + indirects-- + break + } + f.pointers[addr] = f.depth + + ve = ve.Elem() + if ve.Kind() == reflect.Interface { + if ve.IsNil() { + nilFound = true + break + } + ve = ve.Elem() + } + } + + // Display type or indirection level depending on flags. + if showTypes && !f.ignoreNextType { + f.fs.Write(openParenBytes) + f.fs.Write(bytes.Repeat(asteriskBytes, indirects)) + f.fs.Write([]byte(ve.Type().String())) + f.fs.Write(closeParenBytes) + } else { + if nilFound || cycleFound { + indirects += strings.Count(ve.Type().String(), "*") + } + f.fs.Write(openAngleBytes) + f.fs.Write([]byte(strings.Repeat("*", indirects))) + f.fs.Write(closeAngleBytes) + } + + // Display pointer information depending on flags. + if f.fs.Flag('+') && (len(pointerChain) > 0) { + f.fs.Write(openParenBytes) + for i, addr := range pointerChain { + if i > 0 { + f.fs.Write(pointerChainBytes) + } + printHexPtr(f.fs, addr) + } + f.fs.Write(closeParenBytes) + } + + // Display dereferenced value. + switch { + case nilFound: + f.fs.Write(nilAngleBytes) + + case cycleFound: + f.fs.Write(circularShortBytes) + + default: + f.ignoreNextType = true + f.format(ve) + } +} + +// format is the main workhorse for providing the Formatter interface. It +// uses the passed reflect value to figure out what kind of object we are +// dealing with and formats it appropriately. It is a recursive function, +// however circular data structures are detected and handled properly. +func (f *formatState) format(v reflect.Value) { + // Handle invalid reflect values immediately. + kind := v.Kind() + if kind == reflect.Invalid { + f.fs.Write(invalidAngleBytes) + return + } + + // Handle pointers specially. + if kind == reflect.Ptr { + f.formatPtr(v) + return + } + + // Print type information unless already handled elsewhere. + if !f.ignoreNextType && f.fs.Flag('#') { + f.fs.Write(openParenBytes) + f.fs.Write([]byte(v.Type().String())) + f.fs.Write(closeParenBytes) + } + f.ignoreNextType = false + + // Call Stringer/error interfaces if they exist and the handle methods + // flag is enabled. + if !f.cs.DisableMethods { + if (kind != reflect.Invalid) && (kind != reflect.Interface) { + if handled := handleMethods(f.cs, f.fs, v); handled { + return + } + } + } + + switch kind { + case reflect.Invalid: + // Do nothing. We should never get here since invalid has already + // been handled above. + + case reflect.Bool: + printBool(f.fs, v.Bool()) + + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + printInt(f.fs, v.Int(), 10) + + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: + printUint(f.fs, v.Uint(), 10) + + case reflect.Float32: + printFloat(f.fs, v.Float(), 32) + + case reflect.Float64: + printFloat(f.fs, v.Float(), 64) + + case reflect.Complex64: + printComplex(f.fs, v.Complex(), 32) + + case reflect.Complex128: + printComplex(f.fs, v.Complex(), 64) + + case reflect.Slice: + if v.IsNil() { + f.fs.Write(nilAngleBytes) + break + } + fallthrough + + case reflect.Array: + f.fs.Write(openBracketBytes) + f.depth++ + if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { + f.fs.Write(maxShortBytes) + } else { + numEntries := v.Len() + for i := 0; i < numEntries; i++ { + if i > 0 { + f.fs.Write(spaceBytes) + } + f.ignoreNextType = true + f.format(f.unpackValue(v.Index(i))) + } + } + f.depth-- + f.fs.Write(closeBracketBytes) + + case reflect.String: + f.fs.Write([]byte(v.String())) + + case reflect.Interface: + // The only time we should get here is for nil interfaces due to + // unpackValue calls. + if v.IsNil() { + f.fs.Write(nilAngleBytes) + } + + case reflect.Ptr: + // Do nothing. We should never get here since pointers have already + // been handled above. + + case reflect.Map: + // nil maps should be indicated as different than empty maps + if v.IsNil() { + f.fs.Write(nilAngleBytes) + break + } + + f.fs.Write(openMapBytes) + f.depth++ + if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { + f.fs.Write(maxShortBytes) + } else { + keys := v.MapKeys() + if f.cs.SortKeys { + sortValues(keys, f.cs) + } + for i, key := range keys { + if i > 0 { + f.fs.Write(spaceBytes) + } + f.ignoreNextType = true + f.format(f.unpackValue(key)) + f.fs.Write(colonBytes) + f.ignoreNextType = true + f.format(f.unpackValue(v.MapIndex(key))) + } + } + f.depth-- + f.fs.Write(closeMapBytes) + + case reflect.Struct: + numFields := v.NumField() + f.fs.Write(openBraceBytes) + f.depth++ + if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { + f.fs.Write(maxShortBytes) + } else { + vt := v.Type() + for i := 0; i < numFields; i++ { + if i > 0 { + f.fs.Write(spaceBytes) + } + vtf := vt.Field(i) + if f.fs.Flag('+') || f.fs.Flag('#') { + f.fs.Write([]byte(vtf.Name)) + f.fs.Write(colonBytes) + } + f.format(f.unpackValue(v.Field(i))) + } + } + f.depth-- + f.fs.Write(closeBraceBytes) + + case reflect.Uintptr: + printHexPtr(f.fs, uintptr(v.Uint())) + + case reflect.UnsafePointer, reflect.Chan, reflect.Func: + printHexPtr(f.fs, v.Pointer()) + + // There were not any other types at the time this code was written, but + // fall back to letting the default fmt package handle it if any get added. + default: + format := f.buildDefaultFormat() + if v.CanInterface() { + fmt.Fprintf(f.fs, format, v.Interface()) + } else { + fmt.Fprintf(f.fs, format, v.String()) + } + } +} + +// Format satisfies the fmt.Formatter interface. See NewFormatter for usage +// details. +func (f *formatState) Format(fs fmt.State, verb rune) { + f.fs = fs + + // Use standard formatting for verbs that are not v. + if verb != 'v' { + format := f.constructOrigFormat(verb) + fmt.Fprintf(fs, format, f.value) + return + } + + if f.value == nil { + if fs.Flag('#') { + fs.Write(interfaceBytes) + } + fs.Write(nilAngleBytes) + return + } + + f.format(reflect.ValueOf(f.value)) +} + +// newFormatter is a helper function to consolidate the logic from the various +// public methods which take varying config states. +func newFormatter(cs *ConfigState, v interface{}) fmt.Formatter { + fs := &formatState{value: v, cs: cs} + fs.pointers = make(map[uintptr]int) + return fs +} + +/* +NewFormatter returns a custom formatter that satisfies the fmt.Formatter +interface. As a result, it integrates cleanly with standard fmt package +printing functions. The formatter is useful for inline printing of smaller data +types similar to the standard %v format specifier. + +The custom formatter only responds to the %v (most compact), %+v (adds pointer +addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb +combinations. Any other verbs such as %x and %q will be sent to the the +standard fmt package for formatting. In addition, the custom formatter ignores +the width and precision arguments (however they will still work on the format +specifiers not handled by the custom formatter). + +Typically this function shouldn't be called directly. It is much easier to make +use of the custom formatter by calling one of the convenience functions such as +Printf, Println, or Fprintf. +*/ +func NewFormatter(v interface{}) fmt.Formatter { + return newFormatter(&Config, v) +} diff --git a/scraper-go/vendor/github.com/davecgh/go-spew/spew/spew.go b/scraper-go/vendor/github.com/davecgh/go-spew/spew/spew.go new file mode 100644 index 00000000..32c0e338 --- /dev/null +++ b/scraper-go/vendor/github.com/davecgh/go-spew/spew/spew.go @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2013-2016 Dave Collins + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package spew + +import ( + "fmt" + "io" +) + +// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were +// passed with a default Formatter interface returned by NewFormatter. It +// returns the formatted string as a value that satisfies error. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Errorf(format, spew.NewFormatter(a), spew.NewFormatter(b)) +func Errorf(format string, a ...interface{}) (err error) { + return fmt.Errorf(format, convertArgs(a)...) +} + +// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were +// passed with a default Formatter interface returned by NewFormatter. It +// returns the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Fprint(w, spew.NewFormatter(a), spew.NewFormatter(b)) +func Fprint(w io.Writer, a ...interface{}) (n int, err error) { + return fmt.Fprint(w, convertArgs(a)...) +} + +// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were +// passed with a default Formatter interface returned by NewFormatter. It +// returns the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Fprintf(w, format, spew.NewFormatter(a), spew.NewFormatter(b)) +func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(w, format, convertArgs(a)...) +} + +// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it +// passed with a default Formatter interface returned by NewFormatter. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Fprintln(w, spew.NewFormatter(a), spew.NewFormatter(b)) +func Fprintln(w io.Writer, a ...interface{}) (n int, err error) { + return fmt.Fprintln(w, convertArgs(a)...) +} + +// Print is a wrapper for fmt.Print that treats each argument as if it were +// passed with a default Formatter interface returned by NewFormatter. It +// returns the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Print(spew.NewFormatter(a), spew.NewFormatter(b)) +func Print(a ...interface{}) (n int, err error) { + return fmt.Print(convertArgs(a)...) +} + +// Printf is a wrapper for fmt.Printf that treats each argument as if it were +// passed with a default Formatter interface returned by NewFormatter. It +// returns the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Printf(format, spew.NewFormatter(a), spew.NewFormatter(b)) +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Printf(format, convertArgs(a)...) +} + +// Println is a wrapper for fmt.Println that treats each argument as if it were +// passed with a default Formatter interface returned by NewFormatter. It +// returns the number of bytes written and any write error encountered. See +// NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Println(spew.NewFormatter(a), spew.NewFormatter(b)) +func Println(a ...interface{}) (n int, err error) { + return fmt.Println(convertArgs(a)...) +} + +// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were +// passed with a default Formatter interface returned by NewFormatter. It +// returns the resulting string. See NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Sprint(spew.NewFormatter(a), spew.NewFormatter(b)) +func Sprint(a ...interface{}) string { + return fmt.Sprint(convertArgs(a)...) +} + +// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were +// passed with a default Formatter interface returned by NewFormatter. It +// returns the resulting string. See NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Sprintf(format, spew.NewFormatter(a), spew.NewFormatter(b)) +func Sprintf(format string, a ...interface{}) string { + return fmt.Sprintf(format, convertArgs(a)...) +} + +// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it +// were passed with a default Formatter interface returned by NewFormatter. It +// returns the resulting string. See NewFormatter for formatting details. +// +// This function is shorthand for the following syntax: +// +// fmt.Sprintln(spew.NewFormatter(a), spew.NewFormatter(b)) +func Sprintln(a ...interface{}) string { + return fmt.Sprintln(convertArgs(a)...) +} + +// convertArgs accepts a slice of arguments and returns a slice of the same +// length with each argument converted to a default spew Formatter interface. +func convertArgs(args []interface{}) (formatters []interface{}) { + formatters = make([]interface{}, len(args)) + for index, arg := range args { + formatters[index] = NewFormatter(arg) + } + return formatters +} diff --git a/scraper-go/vendor/github.com/pmezard/go-difflib/LICENSE b/scraper-go/vendor/github.com/pmezard/go-difflib/LICENSE new file mode 100644 index 00000000..c67dad61 --- /dev/null +++ b/scraper-go/vendor/github.com/pmezard/go-difflib/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013, Patrick Mezard +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + The names of its contributors may not be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/scraper-go/vendor/github.com/pmezard/go-difflib/difflib/difflib.go b/scraper-go/vendor/github.com/pmezard/go-difflib/difflib/difflib.go new file mode 100644 index 00000000..003e99fa --- /dev/null +++ b/scraper-go/vendor/github.com/pmezard/go-difflib/difflib/difflib.go @@ -0,0 +1,772 @@ +// Package difflib is a partial port of Python difflib module. +// +// It provides tools to compare sequences of strings and generate textual diffs. +// +// The following class and functions have been ported: +// +// - SequenceMatcher +// +// - unified_diff +// +// - context_diff +// +// Getting unified diffs was the main goal of the port. Keep in mind this code +// is mostly suitable to output text differences in a human friendly way, there +// are no guarantees generated diffs are consumable by patch(1). +package difflib + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" +) + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func calculateRatio(matches, length int) float64 { + if length > 0 { + return 2.0 * float64(matches) / float64(length) + } + return 1.0 +} + +type Match struct { + A int + B int + Size int +} + +type OpCode struct { + Tag byte + I1 int + I2 int + J1 int + J2 int +} + +// SequenceMatcher compares sequence of strings. The basic +// algorithm predates, and is a little fancier than, an algorithm +// published in the late 1980's by Ratcliff and Obershelp under the +// hyperbolic name "gestalt pattern matching". The basic idea is to find +// the longest contiguous matching subsequence that contains no "junk" +// elements (R-O doesn't address junk). The same idea is then applied +// recursively to the pieces of the sequences to the left and to the right +// of the matching subsequence. This does not yield minimal edit +// sequences, but does tend to yield matches that "look right" to people. +// +// SequenceMatcher tries to compute a "human-friendly diff" between two +// sequences. Unlike e.g. UNIX(tm) diff, the fundamental notion is the +// longest *contiguous* & junk-free matching subsequence. That's what +// catches peoples' eyes. The Windows(tm) windiff has another interesting +// notion, pairing up elements that appear uniquely in each sequence. +// That, and the method here, appear to yield more intuitive difference +// reports than does diff. This method appears to be the least vulnerable +// to synching up on blocks of "junk lines", though (like blank lines in +// ordinary text files, or maybe "

" lines in HTML files). That may be +// because this is the only method of the 3 that has a *concept* of +// "junk" . +// +// Timing: Basic R-O is cubic time worst case and quadratic time expected +// case. SequenceMatcher is quadratic time for the worst case and has +// expected-case behavior dependent in a complicated way on how many +// elements the sequences have in common; best case time is linear. +type SequenceMatcher struct { + a []string + b []string + b2j map[string][]int + IsJunk func(string) bool + autoJunk bool + bJunk map[string]struct{} + matchingBlocks []Match + fullBCount map[string]int + bPopular map[string]struct{} + opCodes []OpCode +} + +func NewMatcher(a, b []string) *SequenceMatcher { + m := SequenceMatcher{autoJunk: true} + m.SetSeqs(a, b) + return &m +} + +func NewMatcherWithJunk(a, b []string, autoJunk bool, + isJunk func(string) bool) *SequenceMatcher { + + m := SequenceMatcher{IsJunk: isJunk, autoJunk: autoJunk} + m.SetSeqs(a, b) + return &m +} + +// Set two sequences to be compared. +func (m *SequenceMatcher) SetSeqs(a, b []string) { + m.SetSeq1(a) + m.SetSeq2(b) +} + +// Set the first sequence to be compared. The second sequence to be compared is +// not changed. +// +// SequenceMatcher computes and caches detailed information about the second +// sequence, so if you want to compare one sequence S against many sequences, +// use .SetSeq2(s) once and call .SetSeq1(x) repeatedly for each of the other +// sequences. +// +// See also SetSeqs() and SetSeq2(). +func (m *SequenceMatcher) SetSeq1(a []string) { + if &a == &m.a { + return + } + m.a = a + m.matchingBlocks = nil + m.opCodes = nil +} + +// Set the second sequence to be compared. The first sequence to be compared is +// not changed. +func (m *SequenceMatcher) SetSeq2(b []string) { + if &b == &m.b { + return + } + m.b = b + m.matchingBlocks = nil + m.opCodes = nil + m.fullBCount = nil + m.chainB() +} + +func (m *SequenceMatcher) chainB() { + // Populate line -> index mapping + b2j := map[string][]int{} + for i, s := range m.b { + indices := b2j[s] + indices = append(indices, i) + b2j[s] = indices + } + + // Purge junk elements + m.bJunk = map[string]struct{}{} + if m.IsJunk != nil { + junk := m.bJunk + for s, _ := range b2j { + if m.IsJunk(s) { + junk[s] = struct{}{} + } + } + for s, _ := range junk { + delete(b2j, s) + } + } + + // Purge remaining popular elements + popular := map[string]struct{}{} + n := len(m.b) + if m.autoJunk && n >= 200 { + ntest := n/100 + 1 + for s, indices := range b2j { + if len(indices) > ntest { + popular[s] = struct{}{} + } + } + for s, _ := range popular { + delete(b2j, s) + } + } + m.bPopular = popular + m.b2j = b2j +} + +func (m *SequenceMatcher) isBJunk(s string) bool { + _, ok := m.bJunk[s] + return ok +} + +// Find longest matching block in a[alo:ahi] and b[blo:bhi]. +// +// If IsJunk is not defined: +// +// Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where +// alo <= i <= i+k <= ahi +// blo <= j <= j+k <= bhi +// and for all (i',j',k') meeting those conditions, +// k >= k' +// i <= i' +// and if i == i', j <= j' +// +// In other words, of all maximal matching blocks, return one that +// starts earliest in a, and of all those maximal matching blocks that +// start earliest in a, return the one that starts earliest in b. +// +// If IsJunk is defined, first the longest matching block is +// determined as above, but with the additional restriction that no +// junk element appears in the block. Then that block is extended as +// far as possible by matching (only) junk elements on both sides. So +// the resulting block never matches on junk except as identical junk +// happens to be adjacent to an "interesting" match. +// +// If no blocks match, return (alo, blo, 0). +func (m *SequenceMatcher) findLongestMatch(alo, ahi, blo, bhi int) Match { + // CAUTION: stripping common prefix or suffix would be incorrect. + // E.g., + // ab + // acab + // Longest matching block is "ab", but if common prefix is + // stripped, it's "a" (tied with "b"). UNIX(tm) diff does so + // strip, so ends up claiming that ab is changed to acab by + // inserting "ca" in the middle. That's minimal but unintuitive: + // "it's obvious" that someone inserted "ac" at the front. + // Windiff ends up at the same place as diff, but by pairing up + // the unique 'b's and then matching the first two 'a's. + besti, bestj, bestsize := alo, blo, 0 + + // find longest junk-free match + // during an iteration of the loop, j2len[j] = length of longest + // junk-free match ending with a[i-1] and b[j] + j2len := map[int]int{} + for i := alo; i != ahi; i++ { + // look at all instances of a[i] in b; note that because + // b2j has no junk keys, the loop is skipped if a[i] is junk + newj2len := map[int]int{} + for _, j := range m.b2j[m.a[i]] { + // a[i] matches b[j] + if j < blo { + continue + } + if j >= bhi { + break + } + k := j2len[j-1] + 1 + newj2len[j] = k + if k > bestsize { + besti, bestj, bestsize = i-k+1, j-k+1, k + } + } + j2len = newj2len + } + + // Extend the best by non-junk elements on each end. In particular, + // "popular" non-junk elements aren't in b2j, which greatly speeds + // the inner loop above, but also means "the best" match so far + // doesn't contain any junk *or* popular non-junk elements. + for besti > alo && bestj > blo && !m.isBJunk(m.b[bestj-1]) && + m.a[besti-1] == m.b[bestj-1] { + besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 + } + for besti+bestsize < ahi && bestj+bestsize < bhi && + !m.isBJunk(m.b[bestj+bestsize]) && + m.a[besti+bestsize] == m.b[bestj+bestsize] { + bestsize += 1 + } + + // Now that we have a wholly interesting match (albeit possibly + // empty!), we may as well suck up the matching junk on each + // side of it too. Can't think of a good reason not to, and it + // saves post-processing the (possibly considerable) expense of + // figuring out what to do with it. In the case of an empty + // interesting match, this is clearly the right thing to do, + // because no other kind of match is possible in the regions. + for besti > alo && bestj > blo && m.isBJunk(m.b[bestj-1]) && + m.a[besti-1] == m.b[bestj-1] { + besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 + } + for besti+bestsize < ahi && bestj+bestsize < bhi && + m.isBJunk(m.b[bestj+bestsize]) && + m.a[besti+bestsize] == m.b[bestj+bestsize] { + bestsize += 1 + } + + return Match{A: besti, B: bestj, Size: bestsize} +} + +// Return list of triples describing matching subsequences. +// +// Each triple is of the form (i, j, n), and means that +// a[i:i+n] == b[j:j+n]. The triples are monotonically increasing in +// i and in j. It's also guaranteed that if (i, j, n) and (i', j', n') are +// adjacent triples in the list, and the second is not the last triple in the +// list, then i+n != i' or j+n != j'. IOW, adjacent triples never describe +// adjacent equal blocks. +// +// The last triple is a dummy, (len(a), len(b), 0), and is the only +// triple with n==0. +func (m *SequenceMatcher) GetMatchingBlocks() []Match { + if m.matchingBlocks != nil { + return m.matchingBlocks + } + + var matchBlocks func(alo, ahi, blo, bhi int, matched []Match) []Match + matchBlocks = func(alo, ahi, blo, bhi int, matched []Match) []Match { + match := m.findLongestMatch(alo, ahi, blo, bhi) + i, j, k := match.A, match.B, match.Size + if match.Size > 0 { + if alo < i && blo < j { + matched = matchBlocks(alo, i, blo, j, matched) + } + matched = append(matched, match) + if i+k < ahi && j+k < bhi { + matched = matchBlocks(i+k, ahi, j+k, bhi, matched) + } + } + return matched + } + matched := matchBlocks(0, len(m.a), 0, len(m.b), nil) + + // It's possible that we have adjacent equal blocks in the + // matching_blocks list now. + nonAdjacent := []Match{} + i1, j1, k1 := 0, 0, 0 + for _, b := range matched { + // Is this block adjacent to i1, j1, k1? + i2, j2, k2 := b.A, b.B, b.Size + if i1+k1 == i2 && j1+k1 == j2 { + // Yes, so collapse them -- this just increases the length of + // the first block by the length of the second, and the first + // block so lengthened remains the block to compare against. + k1 += k2 + } else { + // Not adjacent. Remember the first block (k1==0 means it's + // the dummy we started with), and make the second block the + // new block to compare against. + if k1 > 0 { + nonAdjacent = append(nonAdjacent, Match{i1, j1, k1}) + } + i1, j1, k1 = i2, j2, k2 + } + } + if k1 > 0 { + nonAdjacent = append(nonAdjacent, Match{i1, j1, k1}) + } + + nonAdjacent = append(nonAdjacent, Match{len(m.a), len(m.b), 0}) + m.matchingBlocks = nonAdjacent + return m.matchingBlocks +} + +// Return list of 5-tuples describing how to turn a into b. +// +// Each tuple is of the form (tag, i1, i2, j1, j2). The first tuple +// has i1 == j1 == 0, and remaining tuples have i1 == the i2 from the +// tuple preceding it, and likewise for j1 == the previous j2. +// +// The tags are characters, with these meanings: +// +// 'r' (replace): a[i1:i2] should be replaced by b[j1:j2] +// +// 'd' (delete): a[i1:i2] should be deleted, j1==j2 in this case. +// +// 'i' (insert): b[j1:j2] should be inserted at a[i1:i1], i1==i2 in this case. +// +// 'e' (equal): a[i1:i2] == b[j1:j2] +func (m *SequenceMatcher) GetOpCodes() []OpCode { + if m.opCodes != nil { + return m.opCodes + } + i, j := 0, 0 + matching := m.GetMatchingBlocks() + opCodes := make([]OpCode, 0, len(matching)) + for _, m := range matching { + // invariant: we've pumped out correct diffs to change + // a[:i] into b[:j], and the next matching block is + // a[ai:ai+size] == b[bj:bj+size]. So we need to pump + // out a diff to change a[i:ai] into b[j:bj], pump out + // the matching block, and move (i,j) beyond the match + ai, bj, size := m.A, m.B, m.Size + tag := byte(0) + if i < ai && j < bj { + tag = 'r' + } else if i < ai { + tag = 'd' + } else if j < bj { + tag = 'i' + } + if tag > 0 { + opCodes = append(opCodes, OpCode{tag, i, ai, j, bj}) + } + i, j = ai+size, bj+size + // the list of matching blocks is terminated by a + // sentinel with size 0 + if size > 0 { + opCodes = append(opCodes, OpCode{'e', ai, i, bj, j}) + } + } + m.opCodes = opCodes + return m.opCodes +} + +// Isolate change clusters by eliminating ranges with no changes. +// +// Return a generator of groups with up to n lines of context. +// Each group is in the same format as returned by GetOpCodes(). +func (m *SequenceMatcher) GetGroupedOpCodes(n int) [][]OpCode { + if n < 0 { + n = 3 + } + codes := m.GetOpCodes() + if len(codes) == 0 { + codes = []OpCode{OpCode{'e', 0, 1, 0, 1}} + } + // Fixup leading and trailing groups if they show no changes. + if codes[0].Tag == 'e' { + c := codes[0] + i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 + codes[0] = OpCode{c.Tag, max(i1, i2-n), i2, max(j1, j2-n), j2} + } + if codes[len(codes)-1].Tag == 'e' { + c := codes[len(codes)-1] + i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 + codes[len(codes)-1] = OpCode{c.Tag, i1, min(i2, i1+n), j1, min(j2, j1+n)} + } + nn := n + n + groups := [][]OpCode{} + group := []OpCode{} + for _, c := range codes { + i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 + // End the current group and start a new one whenever + // there is a large range with no changes. + if c.Tag == 'e' && i2-i1 > nn { + group = append(group, OpCode{c.Tag, i1, min(i2, i1+n), + j1, min(j2, j1+n)}) + groups = append(groups, group) + group = []OpCode{} + i1, j1 = max(i1, i2-n), max(j1, j2-n) + } + group = append(group, OpCode{c.Tag, i1, i2, j1, j2}) + } + if len(group) > 0 && !(len(group) == 1 && group[0].Tag == 'e') { + groups = append(groups, group) + } + return groups +} + +// Return a measure of the sequences' similarity (float in [0,1]). +// +// Where T is the total number of elements in both sequences, and +// M is the number of matches, this is 2.0*M / T. +// Note that this is 1 if the sequences are identical, and 0 if +// they have nothing in common. +// +// .Ratio() is expensive to compute if you haven't already computed +// .GetMatchingBlocks() or .GetOpCodes(), in which case you may +// want to try .QuickRatio() or .RealQuickRation() first to get an +// upper bound. +func (m *SequenceMatcher) Ratio() float64 { + matches := 0 + for _, m := range m.GetMatchingBlocks() { + matches += m.Size + } + return calculateRatio(matches, len(m.a)+len(m.b)) +} + +// Return an upper bound on ratio() relatively quickly. +// +// This isn't defined beyond that it is an upper bound on .Ratio(), and +// is faster to compute. +func (m *SequenceMatcher) QuickRatio() float64 { + // viewing a and b as multisets, set matches to the cardinality + // of their intersection; this counts the number of matches + // without regard to order, so is clearly an upper bound + if m.fullBCount == nil { + m.fullBCount = map[string]int{} + for _, s := range m.b { + m.fullBCount[s] = m.fullBCount[s] + 1 + } + } + + // avail[x] is the number of times x appears in 'b' less the + // number of times we've seen it in 'a' so far ... kinda + avail := map[string]int{} + matches := 0 + for _, s := range m.a { + n, ok := avail[s] + if !ok { + n = m.fullBCount[s] + } + avail[s] = n - 1 + if n > 0 { + matches += 1 + } + } + return calculateRatio(matches, len(m.a)+len(m.b)) +} + +// Return an upper bound on ratio() very quickly. +// +// This isn't defined beyond that it is an upper bound on .Ratio(), and +// is faster to compute than either .Ratio() or .QuickRatio(). +func (m *SequenceMatcher) RealQuickRatio() float64 { + la, lb := len(m.a), len(m.b) + return calculateRatio(min(la, lb), la+lb) +} + +// Convert range to the "ed" format +func formatRangeUnified(start, stop int) string { + // Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning := start + 1 // lines start numbering with one + length := stop - start + if length == 1 { + return fmt.Sprintf("%d", beginning) + } + if length == 0 { + beginning -= 1 // empty ranges begin at line just before the range + } + return fmt.Sprintf("%d,%d", beginning, length) +} + +// Unified diff parameters +type UnifiedDiff struct { + A []string // First sequence lines + FromFile string // First file name + FromDate string // First file time + B []string // Second sequence lines + ToFile string // Second file name + ToDate string // Second file time + Eol string // Headers end of line, defaults to LF + Context int // Number of context lines +} + +// Compare two sequences of lines; generate the delta as a unified diff. +// +// Unified diffs are a compact way of showing line changes and a few +// lines of context. The number of context lines is set by 'n' which +// defaults to three. +// +// By default, the diff control lines (those with ---, +++, or @@) are +// created with a trailing newline. This is helpful so that inputs +// created from file.readlines() result in diffs that are suitable for +// file.writelines() since both the inputs and outputs have trailing +// newlines. +// +// For inputs that do not have trailing newlines, set the lineterm +// argument to "" so that the output will be uniformly newline free. +// +// The unidiff format normally has a header for filenames and modification +// times. Any or all of these may be specified using strings for +// 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. +// The modification times are normally expressed in the ISO 8601 format. +func WriteUnifiedDiff(writer io.Writer, diff UnifiedDiff) error { + buf := bufio.NewWriter(writer) + defer buf.Flush() + wf := func(format string, args ...interface{}) error { + _, err := buf.WriteString(fmt.Sprintf(format, args...)) + return err + } + ws := func(s string) error { + _, err := buf.WriteString(s) + return err + } + + if len(diff.Eol) == 0 { + diff.Eol = "\n" + } + + started := false + m := NewMatcher(diff.A, diff.B) + for _, g := range m.GetGroupedOpCodes(diff.Context) { + if !started { + started = true + fromDate := "" + if len(diff.FromDate) > 0 { + fromDate = "\t" + diff.FromDate + } + toDate := "" + if len(diff.ToDate) > 0 { + toDate = "\t" + diff.ToDate + } + if diff.FromFile != "" || diff.ToFile != "" { + err := wf("--- %s%s%s", diff.FromFile, fromDate, diff.Eol) + if err != nil { + return err + } + err = wf("+++ %s%s%s", diff.ToFile, toDate, diff.Eol) + if err != nil { + return err + } + } + } + first, last := g[0], g[len(g)-1] + range1 := formatRangeUnified(first.I1, last.I2) + range2 := formatRangeUnified(first.J1, last.J2) + if err := wf("@@ -%s +%s @@%s", range1, range2, diff.Eol); err != nil { + return err + } + for _, c := range g { + i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 + if c.Tag == 'e' { + for _, line := range diff.A[i1:i2] { + if err := ws(" " + line); err != nil { + return err + } + } + continue + } + if c.Tag == 'r' || c.Tag == 'd' { + for _, line := range diff.A[i1:i2] { + if err := ws("-" + line); err != nil { + return err + } + } + } + if c.Tag == 'r' || c.Tag == 'i' { + for _, line := range diff.B[j1:j2] { + if err := ws("+" + line); err != nil { + return err + } + } + } + } + } + return nil +} + +// Like WriteUnifiedDiff but returns the diff a string. +func GetUnifiedDiffString(diff UnifiedDiff) (string, error) { + w := &bytes.Buffer{} + err := WriteUnifiedDiff(w, diff) + return string(w.Bytes()), err +} + +// Convert range to the "ed" format. +func formatRangeContext(start, stop int) string { + // Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning := start + 1 // lines start numbering with one + length := stop - start + if length == 0 { + beginning -= 1 // empty ranges begin at line just before the range + } + if length <= 1 { + return fmt.Sprintf("%d", beginning) + } + return fmt.Sprintf("%d,%d", beginning, beginning+length-1) +} + +type ContextDiff UnifiedDiff + +// Compare two sequences of lines; generate the delta as a context diff. +// +// Context diffs are a compact way of showing line changes and a few +// lines of context. The number of context lines is set by diff.Context +// which defaults to three. +// +// By default, the diff control lines (those with *** or ---) are +// created with a trailing newline. +// +// For inputs that do not have trailing newlines, set the diff.Eol +// argument to "" so that the output will be uniformly newline free. +// +// The context diff format normally has a header for filenames and +// modification times. Any or all of these may be specified using +// strings for diff.FromFile, diff.ToFile, diff.FromDate, diff.ToDate. +// The modification times are normally expressed in the ISO 8601 format. +// If not specified, the strings default to blanks. +func WriteContextDiff(writer io.Writer, diff ContextDiff) error { + buf := bufio.NewWriter(writer) + defer buf.Flush() + var diffErr error + wf := func(format string, args ...interface{}) { + _, err := buf.WriteString(fmt.Sprintf(format, args...)) + if diffErr == nil && err != nil { + diffErr = err + } + } + ws := func(s string) { + _, err := buf.WriteString(s) + if diffErr == nil && err != nil { + diffErr = err + } + } + + if len(diff.Eol) == 0 { + diff.Eol = "\n" + } + + prefix := map[byte]string{ + 'i': "+ ", + 'd': "- ", + 'r': "! ", + 'e': " ", + } + + started := false + m := NewMatcher(diff.A, diff.B) + for _, g := range m.GetGroupedOpCodes(diff.Context) { + if !started { + started = true + fromDate := "" + if len(diff.FromDate) > 0 { + fromDate = "\t" + diff.FromDate + } + toDate := "" + if len(diff.ToDate) > 0 { + toDate = "\t" + diff.ToDate + } + if diff.FromFile != "" || diff.ToFile != "" { + wf("*** %s%s%s", diff.FromFile, fromDate, diff.Eol) + wf("--- %s%s%s", diff.ToFile, toDate, diff.Eol) + } + } + + first, last := g[0], g[len(g)-1] + ws("***************" + diff.Eol) + + range1 := formatRangeContext(first.I1, last.I2) + wf("*** %s ****%s", range1, diff.Eol) + for _, c := range g { + if c.Tag == 'r' || c.Tag == 'd' { + for _, cc := range g { + if cc.Tag == 'i' { + continue + } + for _, line := range diff.A[cc.I1:cc.I2] { + ws(prefix[cc.Tag] + line) + } + } + break + } + } + + range2 := formatRangeContext(first.J1, last.J2) + wf("--- %s ----%s", range2, diff.Eol) + for _, c := range g { + if c.Tag == 'r' || c.Tag == 'i' { + for _, cc := range g { + if cc.Tag == 'd' { + continue + } + for _, line := range diff.B[cc.J1:cc.J2] { + ws(prefix[cc.Tag] + line) + } + } + break + } + } + } + return diffErr +} + +// Like WriteContextDiff but returns the diff a string. +func GetContextDiffString(diff ContextDiff) (string, error) { + w := &bytes.Buffer{} + err := WriteContextDiff(w, diff) + return string(w.Bytes()), err +} + +// Split a string on "\n" while preserving them. The output can be used +// as input for UnifiedDiff and ContextDiff structures. +func SplitLines(s string) []string { + lines := strings.SplitAfter(s, "\n") + lines[len(lines)-1] += "\n" + return lines +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/LICENSE b/scraper-go/vendor/github.com/stretchr/testify/LICENSE new file mode 100644 index 00000000..4b0421cf --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_compare.go b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_compare.go new file mode 100644 index 00000000..ffb24e8e --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_compare.go @@ -0,0 +1,495 @@ +package assert + +import ( + "bytes" + "fmt" + "reflect" + "time" +) + +// Deprecated: CompareType has only ever been for internal use and has accidentally been published since v1.6.0. Do not use it. +type CompareType = compareResult + +type compareResult int + +const ( + compareLess compareResult = iota - 1 + compareEqual + compareGreater +) + +var ( + intType = reflect.TypeOf(int(1)) + int8Type = reflect.TypeOf(int8(1)) + int16Type = reflect.TypeOf(int16(1)) + int32Type = reflect.TypeOf(int32(1)) + int64Type = reflect.TypeOf(int64(1)) + + uintType = reflect.TypeOf(uint(1)) + uint8Type = reflect.TypeOf(uint8(1)) + uint16Type = reflect.TypeOf(uint16(1)) + uint32Type = reflect.TypeOf(uint32(1)) + uint64Type = reflect.TypeOf(uint64(1)) + + uintptrType = reflect.TypeOf(uintptr(1)) + + float32Type = reflect.TypeOf(float32(1)) + float64Type = reflect.TypeOf(float64(1)) + + stringType = reflect.TypeOf("") + + timeType = reflect.TypeOf(time.Time{}) + bytesType = reflect.TypeOf([]byte{}) +) + +func compare(obj1, obj2 interface{}, kind reflect.Kind) (compareResult, bool) { + obj1Value := reflect.ValueOf(obj1) + obj2Value := reflect.ValueOf(obj2) + + // throughout this switch we try and avoid calling .Convert() if possible, + // as this has a pretty big performance impact + switch kind { + case reflect.Int: + { + intobj1, ok := obj1.(int) + if !ok { + intobj1 = obj1Value.Convert(intType).Interface().(int) + } + intobj2, ok := obj2.(int) + if !ok { + intobj2 = obj2Value.Convert(intType).Interface().(int) + } + if intobj1 > intobj2 { + return compareGreater, true + } + if intobj1 == intobj2 { + return compareEqual, true + } + if intobj1 < intobj2 { + return compareLess, true + } + } + case reflect.Int8: + { + int8obj1, ok := obj1.(int8) + if !ok { + int8obj1 = obj1Value.Convert(int8Type).Interface().(int8) + } + int8obj2, ok := obj2.(int8) + if !ok { + int8obj2 = obj2Value.Convert(int8Type).Interface().(int8) + } + if int8obj1 > int8obj2 { + return compareGreater, true + } + if int8obj1 == int8obj2 { + return compareEqual, true + } + if int8obj1 < int8obj2 { + return compareLess, true + } + } + case reflect.Int16: + { + int16obj1, ok := obj1.(int16) + if !ok { + int16obj1 = obj1Value.Convert(int16Type).Interface().(int16) + } + int16obj2, ok := obj2.(int16) + if !ok { + int16obj2 = obj2Value.Convert(int16Type).Interface().(int16) + } + if int16obj1 > int16obj2 { + return compareGreater, true + } + if int16obj1 == int16obj2 { + return compareEqual, true + } + if int16obj1 < int16obj2 { + return compareLess, true + } + } + case reflect.Int32: + { + int32obj1, ok := obj1.(int32) + if !ok { + int32obj1 = obj1Value.Convert(int32Type).Interface().(int32) + } + int32obj2, ok := obj2.(int32) + if !ok { + int32obj2 = obj2Value.Convert(int32Type).Interface().(int32) + } + if int32obj1 > int32obj2 { + return compareGreater, true + } + if int32obj1 == int32obj2 { + return compareEqual, true + } + if int32obj1 < int32obj2 { + return compareLess, true + } + } + case reflect.Int64: + { + int64obj1, ok := obj1.(int64) + if !ok { + int64obj1 = obj1Value.Convert(int64Type).Interface().(int64) + } + int64obj2, ok := obj2.(int64) + if !ok { + int64obj2 = obj2Value.Convert(int64Type).Interface().(int64) + } + if int64obj1 > int64obj2 { + return compareGreater, true + } + if int64obj1 == int64obj2 { + return compareEqual, true + } + if int64obj1 < int64obj2 { + return compareLess, true + } + } + case reflect.Uint: + { + uintobj1, ok := obj1.(uint) + if !ok { + uintobj1 = obj1Value.Convert(uintType).Interface().(uint) + } + uintobj2, ok := obj2.(uint) + if !ok { + uintobj2 = obj2Value.Convert(uintType).Interface().(uint) + } + if uintobj1 > uintobj2 { + return compareGreater, true + } + if uintobj1 == uintobj2 { + return compareEqual, true + } + if uintobj1 < uintobj2 { + return compareLess, true + } + } + case reflect.Uint8: + { + uint8obj1, ok := obj1.(uint8) + if !ok { + uint8obj1 = obj1Value.Convert(uint8Type).Interface().(uint8) + } + uint8obj2, ok := obj2.(uint8) + if !ok { + uint8obj2 = obj2Value.Convert(uint8Type).Interface().(uint8) + } + if uint8obj1 > uint8obj2 { + return compareGreater, true + } + if uint8obj1 == uint8obj2 { + return compareEqual, true + } + if uint8obj1 < uint8obj2 { + return compareLess, true + } + } + case reflect.Uint16: + { + uint16obj1, ok := obj1.(uint16) + if !ok { + uint16obj1 = obj1Value.Convert(uint16Type).Interface().(uint16) + } + uint16obj2, ok := obj2.(uint16) + if !ok { + uint16obj2 = obj2Value.Convert(uint16Type).Interface().(uint16) + } + if uint16obj1 > uint16obj2 { + return compareGreater, true + } + if uint16obj1 == uint16obj2 { + return compareEqual, true + } + if uint16obj1 < uint16obj2 { + return compareLess, true + } + } + case reflect.Uint32: + { + uint32obj1, ok := obj1.(uint32) + if !ok { + uint32obj1 = obj1Value.Convert(uint32Type).Interface().(uint32) + } + uint32obj2, ok := obj2.(uint32) + if !ok { + uint32obj2 = obj2Value.Convert(uint32Type).Interface().(uint32) + } + if uint32obj1 > uint32obj2 { + return compareGreater, true + } + if uint32obj1 == uint32obj2 { + return compareEqual, true + } + if uint32obj1 < uint32obj2 { + return compareLess, true + } + } + case reflect.Uint64: + { + uint64obj1, ok := obj1.(uint64) + if !ok { + uint64obj1 = obj1Value.Convert(uint64Type).Interface().(uint64) + } + uint64obj2, ok := obj2.(uint64) + if !ok { + uint64obj2 = obj2Value.Convert(uint64Type).Interface().(uint64) + } + if uint64obj1 > uint64obj2 { + return compareGreater, true + } + if uint64obj1 == uint64obj2 { + return compareEqual, true + } + if uint64obj1 < uint64obj2 { + return compareLess, true + } + } + case reflect.Float32: + { + float32obj1, ok := obj1.(float32) + if !ok { + float32obj1 = obj1Value.Convert(float32Type).Interface().(float32) + } + float32obj2, ok := obj2.(float32) + if !ok { + float32obj2 = obj2Value.Convert(float32Type).Interface().(float32) + } + if float32obj1 > float32obj2 { + return compareGreater, true + } + if float32obj1 == float32obj2 { + return compareEqual, true + } + if float32obj1 < float32obj2 { + return compareLess, true + } + } + case reflect.Float64: + { + float64obj1, ok := obj1.(float64) + if !ok { + float64obj1 = obj1Value.Convert(float64Type).Interface().(float64) + } + float64obj2, ok := obj2.(float64) + if !ok { + float64obj2 = obj2Value.Convert(float64Type).Interface().(float64) + } + if float64obj1 > float64obj2 { + return compareGreater, true + } + if float64obj1 == float64obj2 { + return compareEqual, true + } + if float64obj1 < float64obj2 { + return compareLess, true + } + } + case reflect.String: + { + stringobj1, ok := obj1.(string) + if !ok { + stringobj1 = obj1Value.Convert(stringType).Interface().(string) + } + stringobj2, ok := obj2.(string) + if !ok { + stringobj2 = obj2Value.Convert(stringType).Interface().(string) + } + if stringobj1 > stringobj2 { + return compareGreater, true + } + if stringobj1 == stringobj2 { + return compareEqual, true + } + if stringobj1 < stringobj2 { + return compareLess, true + } + } + // Check for known struct types we can check for compare results. + case reflect.Struct: + { + // All structs enter here. We're not interested in most types. + if !obj1Value.CanConvert(timeType) { + break + } + + // time.Time can be compared! + timeObj1, ok := obj1.(time.Time) + if !ok { + timeObj1 = obj1Value.Convert(timeType).Interface().(time.Time) + } + + timeObj2, ok := obj2.(time.Time) + if !ok { + timeObj2 = obj2Value.Convert(timeType).Interface().(time.Time) + } + + if timeObj1.Before(timeObj2) { + return compareLess, true + } + if timeObj1.Equal(timeObj2) { + return compareEqual, true + } + return compareGreater, true + } + case reflect.Slice: + { + // We only care about the []byte type. + if !obj1Value.CanConvert(bytesType) { + break + } + + // []byte can be compared! + bytesObj1, ok := obj1.([]byte) + if !ok { + bytesObj1 = obj1Value.Convert(bytesType).Interface().([]byte) + + } + bytesObj2, ok := obj2.([]byte) + if !ok { + bytesObj2 = obj2Value.Convert(bytesType).Interface().([]byte) + } + + return compareResult(bytes.Compare(bytesObj1, bytesObj2)), true + } + case reflect.Uintptr: + { + uintptrObj1, ok := obj1.(uintptr) + if !ok { + uintptrObj1 = obj1Value.Convert(uintptrType).Interface().(uintptr) + } + uintptrObj2, ok := obj2.(uintptr) + if !ok { + uintptrObj2 = obj2Value.Convert(uintptrType).Interface().(uintptr) + } + if uintptrObj1 > uintptrObj2 { + return compareGreater, true + } + if uintptrObj1 == uintptrObj2 { + return compareEqual, true + } + if uintptrObj1 < uintptrObj2 { + return compareLess, true + } + } + } + + return compareEqual, false +} + +// Greater asserts that the first element is greater than the second +// +// assert.Greater(t, 2, 1) +// assert.Greater(t, float64(2), float64(1)) +// assert.Greater(t, "b", "a") +func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + failMessage := fmt.Sprintf("\"%v\" is not greater than \"%v\"", e1, e2) + return compareTwoValues(t, e1, e2, []compareResult{compareGreater}, failMessage, msgAndArgs...) +} + +// GreaterOrEqual asserts that the first element is greater than or equal to the second +// +// assert.GreaterOrEqual(t, 2, 1) +// assert.GreaterOrEqual(t, 2, 2) +// assert.GreaterOrEqual(t, "b", "a") +// assert.GreaterOrEqual(t, "b", "b") +func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + failMessage := fmt.Sprintf("\"%v\" is not greater than or equal to \"%v\"", e1, e2) + return compareTwoValues(t, e1, e2, []compareResult{compareGreater, compareEqual}, failMessage, msgAndArgs...) +} + +// Less asserts that the first element is less than the second +// +// assert.Less(t, 1, 2) +// assert.Less(t, float64(1), float64(2)) +// assert.Less(t, "a", "b") +func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + failMessage := fmt.Sprintf("\"%v\" is not less than \"%v\"", e1, e2) + return compareTwoValues(t, e1, e2, []compareResult{compareLess}, failMessage, msgAndArgs...) +} + +// LessOrEqual asserts that the first element is less than or equal to the second +// +// assert.LessOrEqual(t, 1, 2) +// assert.LessOrEqual(t, 2, 2) +// assert.LessOrEqual(t, "a", "b") +// assert.LessOrEqual(t, "b", "b") +func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + failMessage := fmt.Sprintf("\"%v\" is not less than or equal to \"%v\"", e1, e2) + return compareTwoValues(t, e1, e2, []compareResult{compareLess, compareEqual}, failMessage, msgAndArgs...) +} + +// Positive asserts that the specified element is positive +// +// assert.Positive(t, 1) +// assert.Positive(t, 1.23) +func Positive(t TestingT, e interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + zero := reflect.Zero(reflect.TypeOf(e)) + failMessage := fmt.Sprintf("\"%v\" is not positive", e) + return compareTwoValues(t, e, zero.Interface(), []compareResult{compareGreater}, failMessage, msgAndArgs...) +} + +// Negative asserts that the specified element is negative +// +// assert.Negative(t, -1) +// assert.Negative(t, -1.23) +func Negative(t TestingT, e interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + zero := reflect.Zero(reflect.TypeOf(e)) + failMessage := fmt.Sprintf("\"%v\" is not negative", e) + return compareTwoValues(t, e, zero.Interface(), []compareResult{compareLess}, failMessage, msgAndArgs...) +} + +func compareTwoValues(t TestingT, e1 interface{}, e2 interface{}, allowedComparesResults []compareResult, failMessage string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + e1Kind := reflect.ValueOf(e1).Kind() + e2Kind := reflect.ValueOf(e2).Kind() + if e1Kind != e2Kind { + return Fail(t, "Elements should be the same type", msgAndArgs...) + } + + compareResult, isComparable := compare(e1, e2, e1Kind) + if !isComparable { + return Fail(t, fmt.Sprintf(`Can not compare type "%T"`, e1), msgAndArgs...) + } + + if !containsValue(allowedComparesResults, compareResult) { + return Fail(t, failMessage, msgAndArgs...) + } + + return true +} + +func containsValue(values []compareResult, value compareResult) bool { + for _, v := range values { + if v == value { + return true + } + } + + return false +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_format.go b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_format.go new file mode 100644 index 00000000..c592f6ad --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_format.go @@ -0,0 +1,866 @@ +// Code generated with github.com/stretchr/testify/_codegen; DO NOT EDIT. + +package assert + +import ( + http "net/http" + url "net/url" + time "time" +) + +// Conditionf uses a Comparison to assert a complex condition. +func Conditionf(t TestingT, comp Comparison, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Condition(t, comp, append([]interface{}{msg}, args...)...) +} + +// Containsf asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// assert.Containsf(t, "Hello World", "World", "error message %s", "formatted") +// assert.Containsf(t, ["Hello", "World"], "World", "error message %s", "formatted") +// assert.Containsf(t, {"Hello": "World"}, "Hello", "error message %s", "formatted") +func Containsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Contains(t, s, contains, append([]interface{}{msg}, args...)...) +} + +// DirExistsf checks whether a directory exists in the given path. It also fails +// if the path is a file rather a directory or there is an error checking whether it exists. +func DirExistsf(t TestingT, path string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return DirExists(t, path, append([]interface{}{msg}, args...)...) +} + +// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// assert.ElementsMatchf(t, [1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") +func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return ElementsMatch(t, listA, listB, append([]interface{}{msg}, args...)...) +} + +// Emptyf asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". +// +// assert.Emptyf(t, obj, "error message %s", "formatted") +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value +func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Empty(t, object, append([]interface{}{msg}, args...)...) +} + +// Equalf asserts that two objects are equal. +// +// assert.Equalf(t, 123, 123, "error message %s", "formatted") +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Equal(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// EqualErrorf asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// assert.EqualErrorf(t, err, expectedErrorString, "error message %s", "formatted") +func EqualErrorf(t TestingT, theError error, errString string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return EqualError(t, theError, errString, append([]interface{}{msg}, args...)...) +} + +// EqualExportedValuesf asserts that the types of two objects are equal and their public +// fields are also equal. This is useful for comparing structs that have private fields +// that could potentially differ. +// +// type S struct { +// Exported int +// notExported int +// } +// assert.EqualExportedValuesf(t, S{1, 2}, S{1, 3}, "error message %s", "formatted") => true +// assert.EqualExportedValuesf(t, S{1, 2}, S{2, 3}, "error message %s", "formatted") => false +func EqualExportedValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return EqualExportedValues(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// EqualValuesf asserts that two objects are equal or convertible to the larger +// type and equal. +// +// assert.EqualValuesf(t, uint32(123), int32(123), "error message %s", "formatted") +func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return EqualValues(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// Errorf asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// assert.Errorf(t, err, "error message %s", "formatted") +func Errorf(t TestingT, err error, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Error(t, err, append([]interface{}{msg}, args...)...) +} + +// ErrorAsf asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. +// This is a wrapper for errors.As. +func ErrorAsf(t TestingT, err error, target interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return ErrorAs(t, err, target, append([]interface{}{msg}, args...)...) +} + +// ErrorContainsf asserts that a function returned an error (i.e. not `nil`) +// and that the error contains the specified substring. +// +// actualObj, err := SomeFunction() +// assert.ErrorContainsf(t, err, expectedErrorSubString, "error message %s", "formatted") +func ErrorContainsf(t TestingT, theError error, contains string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return ErrorContains(t, theError, contains, append([]interface{}{msg}, args...)...) +} + +// ErrorIsf asserts that at least one of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func ErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return ErrorIs(t, err, target, append([]interface{}{msg}, args...)...) +} + +// Eventuallyf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// assert.Eventuallyf(t, func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func Eventuallyf(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Eventually(t, condition, waitFor, tick, append([]interface{}{msg}, args...)...) +} + +// EventuallyWithTf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. In contrast to Eventually, +// it supplies a CollectT to the condition function, so that the condition +// function can use the CollectT to call other assertions. +// The condition is considered "met" if no errors are raised in a tick. +// The supplied CollectT collects all errors from one tick (if there are any). +// If the condition is not met before waitFor, the collected errors of +// the last tick are copied to t. +// +// externalValue := false +// go func() { +// time.Sleep(8*time.Second) +// externalValue = true +// }() +// assert.EventuallyWithTf(t, func(c *assert.CollectT, "error message %s", "formatted") { +// // add assertions as needed; any assertion failure will fail the current tick +// assert.True(c, externalValue, "expected 'externalValue' to be true") +// }, 10*time.Second, 1*time.Second, "external state has not changed to 'true'; still false") +func EventuallyWithTf(t TestingT, condition func(collect *CollectT), waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return EventuallyWithT(t, condition, waitFor, tick, append([]interface{}{msg}, args...)...) +} + +// Exactlyf asserts that two objects are equal in value and type. +// +// assert.Exactlyf(t, int32(123), int64(123), "error message %s", "formatted") +func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Exactly(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// Failf reports a failure through +func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, failureMessage, append([]interface{}{msg}, args...)...) +} + +// FailNowf fails test +func FailNowf(t TestingT, failureMessage string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return FailNow(t, failureMessage, append([]interface{}{msg}, args...)...) +} + +// Falsef asserts that the specified value is false. +// +// assert.Falsef(t, myBool, "error message %s", "formatted") +func Falsef(t TestingT, value bool, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return False(t, value, append([]interface{}{msg}, args...)...) +} + +// FileExistsf checks whether a file exists in the given path. It also fails if +// the path points to a directory or there is an error when trying to check the file. +func FileExistsf(t TestingT, path string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return FileExists(t, path, append([]interface{}{msg}, args...)...) +} + +// Greaterf asserts that the first element is greater than the second +// +// assert.Greaterf(t, 2, 1, "error message %s", "formatted") +// assert.Greaterf(t, float64(2), float64(1), "error message %s", "formatted") +// assert.Greaterf(t, "b", "a", "error message %s", "formatted") +func Greaterf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Greater(t, e1, e2, append([]interface{}{msg}, args...)...) +} + +// GreaterOrEqualf asserts that the first element is greater than or equal to the second +// +// assert.GreaterOrEqualf(t, 2, 1, "error message %s", "formatted") +// assert.GreaterOrEqualf(t, 2, 2, "error message %s", "formatted") +// assert.GreaterOrEqualf(t, "b", "a", "error message %s", "formatted") +// assert.GreaterOrEqualf(t, "b", "b", "error message %s", "formatted") +func GreaterOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return GreaterOrEqual(t, e1, e2, append([]interface{}{msg}, args...)...) +} + +// HTTPBodyContainsf asserts that a specified handler returns a +// body that contains a string. +// +// assert.HTTPBodyContainsf(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPBodyContains(t, handler, method, url, values, str, append([]interface{}{msg}, args...)...) +} + +// HTTPBodyNotContainsf asserts that a specified handler returns a +// body that does not contain a string. +// +// assert.HTTPBodyNotContainsf(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPBodyNotContains(t, handler, method, url, values, str, append([]interface{}{msg}, args...)...) +} + +// HTTPErrorf asserts that a specified handler returns an error status code. +// +// assert.HTTPErrorf(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPError(t, handler, method, url, values, append([]interface{}{msg}, args...)...) +} + +// HTTPRedirectf asserts that a specified handler returns a redirect status code. +// +// assert.HTTPRedirectf(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPRedirect(t, handler, method, url, values, append([]interface{}{msg}, args...)...) +} + +// HTTPStatusCodef asserts that a specified handler returns a specified status code. +// +// assert.HTTPStatusCodef(t, myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPStatusCodef(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPStatusCode(t, handler, method, url, values, statuscode, append([]interface{}{msg}, args...)...) +} + +// HTTPSuccessf asserts that a specified handler returns a success status code. +// +// assert.HTTPSuccessf(t, myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPSuccess(t, handler, method, url, values, append([]interface{}{msg}, args...)...) +} + +// Implementsf asserts that an object is implemented by the specified interface. +// +// assert.Implementsf(t, (*MyInterface)(nil), new(MyObject), "error message %s", "formatted") +func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Implements(t, interfaceObject, object, append([]interface{}{msg}, args...)...) +} + +// InDeltaf asserts that the two numerals are within delta of each other. +// +// assert.InDeltaf(t, math.Pi, 22/7.0, 0.01, "error message %s", "formatted") +func InDeltaf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return InDelta(t, expected, actual, delta, append([]interface{}{msg}, args...)...) +} + +// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func InDeltaMapValuesf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return InDeltaMapValues(t, expected, actual, delta, append([]interface{}{msg}, args...)...) +} + +// InDeltaSlicef is the same as InDelta, except it compares two slices. +func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return InDeltaSlice(t, expected, actual, delta, append([]interface{}{msg}, args...)...) +} + +// InEpsilonf asserts that expected and actual have a relative error less than epsilon +func InEpsilonf(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return InEpsilon(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...) +} + +// InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. +func InEpsilonSlicef(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return InEpsilonSlice(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...) +} + +// IsDecreasingf asserts that the collection is decreasing +// +// assert.IsDecreasingf(t, []int{2, 1, 0}, "error message %s", "formatted") +// assert.IsDecreasingf(t, []float{2, 1}, "error message %s", "formatted") +// assert.IsDecreasingf(t, []string{"b", "a"}, "error message %s", "formatted") +func IsDecreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return IsDecreasing(t, object, append([]interface{}{msg}, args...)...) +} + +// IsIncreasingf asserts that the collection is increasing +// +// assert.IsIncreasingf(t, []int{1, 2, 3}, "error message %s", "formatted") +// assert.IsIncreasingf(t, []float{1, 2}, "error message %s", "formatted") +// assert.IsIncreasingf(t, []string{"a", "b"}, "error message %s", "formatted") +func IsIncreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return IsIncreasing(t, object, append([]interface{}{msg}, args...)...) +} + +// IsNonDecreasingf asserts that the collection is not decreasing +// +// assert.IsNonDecreasingf(t, []int{1, 1, 2}, "error message %s", "formatted") +// assert.IsNonDecreasingf(t, []float{1, 2}, "error message %s", "formatted") +// assert.IsNonDecreasingf(t, []string{"a", "b"}, "error message %s", "formatted") +func IsNonDecreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return IsNonDecreasing(t, object, append([]interface{}{msg}, args...)...) +} + +// IsNonIncreasingf asserts that the collection is not increasing +// +// assert.IsNonIncreasingf(t, []int{2, 1, 1}, "error message %s", "formatted") +// assert.IsNonIncreasingf(t, []float{2, 1}, "error message %s", "formatted") +// assert.IsNonIncreasingf(t, []string{"b", "a"}, "error message %s", "formatted") +func IsNonIncreasingf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return IsNonIncreasing(t, object, append([]interface{}{msg}, args...)...) +} + +// IsNotTypef asserts that the specified objects are not of the same type. +// +// assert.IsNotTypef(t, &NotMyStruct{}, &MyStruct{}, "error message %s", "formatted") +func IsNotTypef(t TestingT, theType interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return IsNotType(t, theType, object, append([]interface{}{msg}, args...)...) +} + +// IsTypef asserts that the specified objects are of the same type. +// +// assert.IsTypef(t, &MyStruct{}, &MyStruct{}, "error message %s", "formatted") +func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return IsType(t, expectedType, object, append([]interface{}{msg}, args...)...) +} + +// JSONEqf asserts that two JSON strings are equivalent. +// +// assert.JSONEqf(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") +func JSONEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return JSONEq(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// Lenf asserts that the specified object has specific length. +// Lenf also fails if the object has a type that len() not accept. +// +// assert.Lenf(t, mySlice, 3, "error message %s", "formatted") +func Lenf(t TestingT, object interface{}, length int, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Len(t, object, length, append([]interface{}{msg}, args...)...) +} + +// Lessf asserts that the first element is less than the second +// +// assert.Lessf(t, 1, 2, "error message %s", "formatted") +// assert.Lessf(t, float64(1), float64(2), "error message %s", "formatted") +// assert.Lessf(t, "a", "b", "error message %s", "formatted") +func Lessf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Less(t, e1, e2, append([]interface{}{msg}, args...)...) +} + +// LessOrEqualf asserts that the first element is less than or equal to the second +// +// assert.LessOrEqualf(t, 1, 2, "error message %s", "formatted") +// assert.LessOrEqualf(t, 2, 2, "error message %s", "formatted") +// assert.LessOrEqualf(t, "a", "b", "error message %s", "formatted") +// assert.LessOrEqualf(t, "b", "b", "error message %s", "formatted") +func LessOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return LessOrEqual(t, e1, e2, append([]interface{}{msg}, args...)...) +} + +// Negativef asserts that the specified element is negative +// +// assert.Negativef(t, -1, "error message %s", "formatted") +// assert.Negativef(t, -1.23, "error message %s", "formatted") +func Negativef(t TestingT, e interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Negative(t, e, append([]interface{}{msg}, args...)...) +} + +// Neverf asserts that the given condition doesn't satisfy in waitFor time, +// periodically checking the target function each tick. +// +// assert.Neverf(t, func() bool { return false; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func Neverf(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Never(t, condition, waitFor, tick, append([]interface{}{msg}, args...)...) +} + +// Nilf asserts that the specified object is nil. +// +// assert.Nilf(t, err, "error message %s", "formatted") +func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Nil(t, object, append([]interface{}{msg}, args...)...) +} + +// NoDirExistsf checks whether a directory does not exist in the given path. +// It fails if the path points to an existing _directory_ only. +func NoDirExistsf(t TestingT, path string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NoDirExists(t, path, append([]interface{}{msg}, args...)...) +} + +// NoErrorf asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if assert.NoErrorf(t, err, "error message %s", "formatted") { +// assert.Equal(t, expectedObj, actualObj) +// } +func NoErrorf(t TestingT, err error, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NoError(t, err, append([]interface{}{msg}, args...)...) +} + +// NoFileExistsf checks whether a file does not exist in a given path. It fails +// if the path points to an existing _file_ only. +func NoFileExistsf(t TestingT, path string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NoFileExists(t, path, append([]interface{}{msg}, args...)...) +} + +// NotContainsf asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// assert.NotContainsf(t, "Hello World", "Earth", "error message %s", "formatted") +// assert.NotContainsf(t, ["Hello", "World"], "Earth", "error message %s", "formatted") +// assert.NotContainsf(t, {"Hello": "World"}, "Earth", "error message %s", "formatted") +func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotContains(t, s, contains, append([]interface{}{msg}, args...)...) +} + +// NotElementsMatchf asserts that the specified listA(array, slice...) is NOT equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should not match. +// This is an inverse of ElementsMatch. +// +// assert.NotElementsMatchf(t, [1, 1, 2, 3], [1, 1, 2, 3], "error message %s", "formatted") -> false +// +// assert.NotElementsMatchf(t, [1, 1, 2, 3], [1, 2, 3], "error message %s", "formatted") -> true +// +// assert.NotElementsMatchf(t, [1, 2, 3], [1, 2, 4], "error message %s", "formatted") -> true +func NotElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotElementsMatch(t, listA, listB, append([]interface{}{msg}, args...)...) +} + +// NotEmptyf asserts that the specified object is NOT [Empty]. +// +// if assert.NotEmptyf(t, obj, "error message %s", "formatted") { +// assert.Equal(t, "two", obj[1]) +// } +func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotEmpty(t, object, append([]interface{}{msg}, args...)...) +} + +// NotEqualf asserts that the specified values are NOT equal. +// +// assert.NotEqualf(t, obj1, obj2, "error message %s", "formatted") +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). +func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotEqual(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// NotEqualValuesf asserts that two objects are not equal even when converted to the same type +// +// assert.NotEqualValuesf(t, obj1, obj2, "error message %s", "formatted") +func NotEqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotEqualValues(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// NotErrorAsf asserts that none of the errors in err's chain matches target, +// but if so, sets target to that error value. +func NotErrorAsf(t TestingT, err error, target interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotErrorAs(t, err, target, append([]interface{}{msg}, args...)...) +} + +// NotErrorIsf asserts that none of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func NotErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotErrorIs(t, err, target, append([]interface{}{msg}, args...)...) +} + +// NotImplementsf asserts that an object does not implement the specified interface. +// +// assert.NotImplementsf(t, (*MyInterface)(nil), new(MyObject), "error message %s", "formatted") +func NotImplementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotImplements(t, interfaceObject, object, append([]interface{}{msg}, args...)...) +} + +// NotNilf asserts that the specified object is not nil. +// +// assert.NotNilf(t, err, "error message %s", "formatted") +func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotNil(t, object, append([]interface{}{msg}, args...)...) +} + +// NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// assert.NotPanicsf(t, func(){ RemainCalm() }, "error message %s", "formatted") +func NotPanicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotPanics(t, f, append([]interface{}{msg}, args...)...) +} + +// NotRegexpf asserts that a specified regexp does not match a string. +// +// assert.NotRegexpf(t, regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") +// assert.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted") +func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotRegexp(t, rx, str, append([]interface{}{msg}, args...)...) +} + +// NotSamef asserts that two pointers do not reference the same object. +// +// assert.NotSamef(t, ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func NotSamef(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotSame(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// NotSubsetf asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// assert.NotSubsetf(t, [1, 3, 4], [1, 2], "error message %s", "formatted") +// assert.NotSubsetf(t, {"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") +// assert.NotSubsetf(t, [1, 3, 4], {1: "one", 2: "two"}, "error message %s", "formatted") +// assert.NotSubsetf(t, {"x": 1, "y": 2}, ["z"], "error message %s", "formatted") +func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotSubset(t, list, subset, append([]interface{}{msg}, args...)...) +} + +// NotZerof asserts that i is not the zero value for its type. +func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotZero(t, i, append([]interface{}{msg}, args...)...) +} + +// Panicsf asserts that the code inside the specified PanicTestFunc panics. +// +// assert.Panicsf(t, func(){ GoCrazy() }, "error message %s", "formatted") +func Panicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Panics(t, f, append([]interface{}{msg}, args...)...) +} + +// PanicsWithErrorf asserts that the code inside the specified PanicTestFunc +// panics, and that the recovered panic value is an error that satisfies the +// EqualError comparison. +// +// assert.PanicsWithErrorf(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") +func PanicsWithErrorf(t TestingT, errString string, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return PanicsWithError(t, errString, f, append([]interface{}{msg}, args...)...) +} + +// PanicsWithValuef asserts that the code inside the specified PanicTestFunc panics, and that +// the recovered panic value equals the expected panic value. +// +// assert.PanicsWithValuef(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") +func PanicsWithValuef(t TestingT, expected interface{}, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return PanicsWithValue(t, expected, f, append([]interface{}{msg}, args...)...) +} + +// Positivef asserts that the specified element is positive +// +// assert.Positivef(t, 1, "error message %s", "formatted") +// assert.Positivef(t, 1.23, "error message %s", "formatted") +func Positivef(t TestingT, e interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Positive(t, e, append([]interface{}{msg}, args...)...) +} + +// Regexpf asserts that a specified regexp matches a string. +// +// assert.Regexpf(t, regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") +// assert.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted") +func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Regexp(t, rx, str, append([]interface{}{msg}, args...)...) +} + +// Samef asserts that two pointers reference the same object. +// +// assert.Samef(t, ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func Samef(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Same(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// Subsetf asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// assert.Subsetf(t, [1, 2, 3], [1, 2], "error message %s", "formatted") +// assert.Subsetf(t, {"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") +// assert.Subsetf(t, [1, 2, 3], {1: "one", 2: "two"}, "error message %s", "formatted") +// assert.Subsetf(t, {"x": 1, "y": 2}, ["x"], "error message %s", "formatted") +func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Subset(t, list, subset, append([]interface{}{msg}, args...)...) +} + +// Truef asserts that the specified value is true. +// +// assert.Truef(t, myBool, "error message %s", "formatted") +func Truef(t TestingT, value bool, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return True(t, value, append([]interface{}{msg}, args...)...) +} + +// WithinDurationf asserts that the two times are within duration delta of each other. +// +// assert.WithinDurationf(t, time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") +func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return WithinDuration(t, expected, actual, delta, append([]interface{}{msg}, args...)...) +} + +// WithinRangef asserts that a time is within a time range (inclusive). +// +// assert.WithinRangef(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") +func WithinRangef(t TestingT, actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return WithinRange(t, actual, start, end, append([]interface{}{msg}, args...)...) +} + +// YAMLEqf asserts that two YAML strings are equivalent. +func YAMLEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return YAMLEq(t, expected, actual, append([]interface{}{msg}, args...)...) +} + +// Zerof asserts that i is the zero value for its type. +func Zerof(t TestingT, i interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Zero(t, i, append([]interface{}{msg}, args...)...) +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl new file mode 100644 index 00000000..d2bb0b81 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl @@ -0,0 +1,5 @@ +{{.CommentFormat}} +func {{.DocInfo.Name}}f(t TestingT, {{.ParamsFormat}}) bool { + if h, ok := t.(tHelper); ok { h.Helper() } + return {{.DocInfo.Name}}(t, {{.ForwardedParamsFormat}}) +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_forward.go b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_forward.go new file mode 100644 index 00000000..58db9284 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_forward.go @@ -0,0 +1,1723 @@ +// Code generated with github.com/stretchr/testify/_codegen; DO NOT EDIT. + +package assert + +import ( + http "net/http" + url "net/url" + time "time" +) + +// Condition uses a Comparison to assert a complex condition. +func (a *Assertions) Condition(comp Comparison, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Condition(a.t, comp, msgAndArgs...) +} + +// Conditionf uses a Comparison to assert a complex condition. +func (a *Assertions) Conditionf(comp Comparison, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Conditionf(a.t, comp, msg, args...) +} + +// Contains asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// a.Contains("Hello World", "World") +// a.Contains(["Hello", "World"], "World") +// a.Contains({"Hello": "World"}, "Hello") +func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Contains(a.t, s, contains, msgAndArgs...) +} + +// Containsf asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// a.Containsf("Hello World", "World", "error message %s", "formatted") +// a.Containsf(["Hello", "World"], "World", "error message %s", "formatted") +// a.Containsf({"Hello": "World"}, "Hello", "error message %s", "formatted") +func (a *Assertions) Containsf(s interface{}, contains interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Containsf(a.t, s, contains, msg, args...) +} + +// DirExists checks whether a directory exists in the given path. It also fails +// if the path is a file rather a directory or there is an error checking whether it exists. +func (a *Assertions) DirExists(path string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return DirExists(a.t, path, msgAndArgs...) +} + +// DirExistsf checks whether a directory exists in the given path. It also fails +// if the path is a file rather a directory or there is an error checking whether it exists. +func (a *Assertions) DirExistsf(path string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return DirExistsf(a.t, path, msg, args...) +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// a.ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]) +func (a *Assertions) ElementsMatch(listA interface{}, listB interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ElementsMatch(a.t, listA, listB, msgAndArgs...) +} + +// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// a.ElementsMatchf([1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") +func (a *Assertions) ElementsMatchf(listA interface{}, listB interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ElementsMatchf(a.t, listA, listB, msg, args...) +} + +// Empty asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". +// +// a.Empty(obj) +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value +func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Empty(a.t, object, msgAndArgs...) +} + +// Emptyf asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". +// +// a.Emptyf(obj, "error message %s", "formatted") +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value +func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Emptyf(a.t, object, msg, args...) +} + +// Equal asserts that two objects are equal. +// +// a.Equal(123, 123) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Equal(a.t, expected, actual, msgAndArgs...) +} + +// EqualError asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// a.EqualError(err, expectedErrorString) +func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return EqualError(a.t, theError, errString, msgAndArgs...) +} + +// EqualErrorf asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// a.EqualErrorf(err, expectedErrorString, "error message %s", "formatted") +func (a *Assertions) EqualErrorf(theError error, errString string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return EqualErrorf(a.t, theError, errString, msg, args...) +} + +// EqualExportedValues asserts that the types of two objects are equal and their public +// fields are also equal. This is useful for comparing structs that have private fields +// that could potentially differ. +// +// type S struct { +// Exported int +// notExported int +// } +// a.EqualExportedValues(S{1, 2}, S{1, 3}) => true +// a.EqualExportedValues(S{1, 2}, S{2, 3}) => false +func (a *Assertions) EqualExportedValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return EqualExportedValues(a.t, expected, actual, msgAndArgs...) +} + +// EqualExportedValuesf asserts that the types of two objects are equal and their public +// fields are also equal. This is useful for comparing structs that have private fields +// that could potentially differ. +// +// type S struct { +// Exported int +// notExported int +// } +// a.EqualExportedValuesf(S{1, 2}, S{1, 3}, "error message %s", "formatted") => true +// a.EqualExportedValuesf(S{1, 2}, S{2, 3}, "error message %s", "formatted") => false +func (a *Assertions) EqualExportedValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return EqualExportedValuesf(a.t, expected, actual, msg, args...) +} + +// EqualValues asserts that two objects are equal or convertible to the larger +// type and equal. +// +// a.EqualValues(uint32(123), int32(123)) +func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return EqualValues(a.t, expected, actual, msgAndArgs...) +} + +// EqualValuesf asserts that two objects are equal or convertible to the larger +// type and equal. +// +// a.EqualValuesf(uint32(123), int32(123), "error message %s", "formatted") +func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return EqualValuesf(a.t, expected, actual, msg, args...) +} + +// Equalf asserts that two objects are equal. +// +// a.Equalf(123, 123, "error message %s", "formatted") +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Equalf(a.t, expected, actual, msg, args...) +} + +// Error asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// a.Error(err) +func (a *Assertions) Error(err error, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Error(a.t, err, msgAndArgs...) +} + +// ErrorAs asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. +// This is a wrapper for errors.As. +func (a *Assertions) ErrorAs(err error, target interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ErrorAs(a.t, err, target, msgAndArgs...) +} + +// ErrorAsf asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. +// This is a wrapper for errors.As. +func (a *Assertions) ErrorAsf(err error, target interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ErrorAsf(a.t, err, target, msg, args...) +} + +// ErrorContains asserts that a function returned an error (i.e. not `nil`) +// and that the error contains the specified substring. +// +// actualObj, err := SomeFunction() +// a.ErrorContains(err, expectedErrorSubString) +func (a *Assertions) ErrorContains(theError error, contains string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ErrorContains(a.t, theError, contains, msgAndArgs...) +} + +// ErrorContainsf asserts that a function returned an error (i.e. not `nil`) +// and that the error contains the specified substring. +// +// actualObj, err := SomeFunction() +// a.ErrorContainsf(err, expectedErrorSubString, "error message %s", "formatted") +func (a *Assertions) ErrorContainsf(theError error, contains string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ErrorContainsf(a.t, theError, contains, msg, args...) +} + +// ErrorIs asserts that at least one of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func (a *Assertions) ErrorIs(err error, target error, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ErrorIs(a.t, err, target, msgAndArgs...) +} + +// ErrorIsf asserts that at least one of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func (a *Assertions) ErrorIsf(err error, target error, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ErrorIsf(a.t, err, target, msg, args...) +} + +// Errorf asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// a.Errorf(err, "error message %s", "formatted") +func (a *Assertions) Errorf(err error, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Errorf(a.t, err, msg, args...) +} + +// Eventually asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// a.Eventually(func() bool { return true; }, time.Second, 10*time.Millisecond) +func (a *Assertions) Eventually(condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Eventually(a.t, condition, waitFor, tick, msgAndArgs...) +} + +// EventuallyWithT asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. In contrast to Eventually, +// it supplies a CollectT to the condition function, so that the condition +// function can use the CollectT to call other assertions. +// The condition is considered "met" if no errors are raised in a tick. +// The supplied CollectT collects all errors from one tick (if there are any). +// If the condition is not met before waitFor, the collected errors of +// the last tick are copied to t. +// +// externalValue := false +// go func() { +// time.Sleep(8*time.Second) +// externalValue = true +// }() +// a.EventuallyWithT(func(c *assert.CollectT) { +// // add assertions as needed; any assertion failure will fail the current tick +// assert.True(c, externalValue, "expected 'externalValue' to be true") +// }, 10*time.Second, 1*time.Second, "external state has not changed to 'true'; still false") +func (a *Assertions) EventuallyWithT(condition func(collect *CollectT), waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return EventuallyWithT(a.t, condition, waitFor, tick, msgAndArgs...) +} + +// EventuallyWithTf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. In contrast to Eventually, +// it supplies a CollectT to the condition function, so that the condition +// function can use the CollectT to call other assertions. +// The condition is considered "met" if no errors are raised in a tick. +// The supplied CollectT collects all errors from one tick (if there are any). +// If the condition is not met before waitFor, the collected errors of +// the last tick are copied to t. +// +// externalValue := false +// go func() { +// time.Sleep(8*time.Second) +// externalValue = true +// }() +// a.EventuallyWithTf(func(c *assert.CollectT, "error message %s", "formatted") { +// // add assertions as needed; any assertion failure will fail the current tick +// assert.True(c, externalValue, "expected 'externalValue' to be true") +// }, 10*time.Second, 1*time.Second, "external state has not changed to 'true'; still false") +func (a *Assertions) EventuallyWithTf(condition func(collect *CollectT), waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return EventuallyWithTf(a.t, condition, waitFor, tick, msg, args...) +} + +// Eventuallyf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// a.Eventuallyf(func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func (a *Assertions) Eventuallyf(condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Eventuallyf(a.t, condition, waitFor, tick, msg, args...) +} + +// Exactly asserts that two objects are equal in value and type. +// +// a.Exactly(int32(123), int64(123)) +func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Exactly(a.t, expected, actual, msgAndArgs...) +} + +// Exactlyf asserts that two objects are equal in value and type. +// +// a.Exactlyf(int32(123), int64(123), "error message %s", "formatted") +func (a *Assertions) Exactlyf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Exactlyf(a.t, expected, actual, msg, args...) +} + +// Fail reports a failure through +func (a *Assertions) Fail(failureMessage string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Fail(a.t, failureMessage, msgAndArgs...) +} + +// FailNow fails test +func (a *Assertions) FailNow(failureMessage string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return FailNow(a.t, failureMessage, msgAndArgs...) +} + +// FailNowf fails test +func (a *Assertions) FailNowf(failureMessage string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return FailNowf(a.t, failureMessage, msg, args...) +} + +// Failf reports a failure through +func (a *Assertions) Failf(failureMessage string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Failf(a.t, failureMessage, msg, args...) +} + +// False asserts that the specified value is false. +// +// a.False(myBool) +func (a *Assertions) False(value bool, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return False(a.t, value, msgAndArgs...) +} + +// Falsef asserts that the specified value is false. +// +// a.Falsef(myBool, "error message %s", "formatted") +func (a *Assertions) Falsef(value bool, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Falsef(a.t, value, msg, args...) +} + +// FileExists checks whether a file exists in the given path. It also fails if +// the path points to a directory or there is an error when trying to check the file. +func (a *Assertions) FileExists(path string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return FileExists(a.t, path, msgAndArgs...) +} + +// FileExistsf checks whether a file exists in the given path. It also fails if +// the path points to a directory or there is an error when trying to check the file. +func (a *Assertions) FileExistsf(path string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return FileExistsf(a.t, path, msg, args...) +} + +// Greater asserts that the first element is greater than the second +// +// a.Greater(2, 1) +// a.Greater(float64(2), float64(1)) +// a.Greater("b", "a") +func (a *Assertions) Greater(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Greater(a.t, e1, e2, msgAndArgs...) +} + +// GreaterOrEqual asserts that the first element is greater than or equal to the second +// +// a.GreaterOrEqual(2, 1) +// a.GreaterOrEqual(2, 2) +// a.GreaterOrEqual("b", "a") +// a.GreaterOrEqual("b", "b") +func (a *Assertions) GreaterOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return GreaterOrEqual(a.t, e1, e2, msgAndArgs...) +} + +// GreaterOrEqualf asserts that the first element is greater than or equal to the second +// +// a.GreaterOrEqualf(2, 1, "error message %s", "formatted") +// a.GreaterOrEqualf(2, 2, "error message %s", "formatted") +// a.GreaterOrEqualf("b", "a", "error message %s", "formatted") +// a.GreaterOrEqualf("b", "b", "error message %s", "formatted") +func (a *Assertions) GreaterOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return GreaterOrEqualf(a.t, e1, e2, msg, args...) +} + +// Greaterf asserts that the first element is greater than the second +// +// a.Greaterf(2, 1, "error message %s", "formatted") +// a.Greaterf(float64(2), float64(1), "error message %s", "formatted") +// a.Greaterf("b", "a", "error message %s", "formatted") +func (a *Assertions) Greaterf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Greaterf(a.t, e1, e2, msg, args...) +} + +// HTTPBodyContains asserts that a specified handler returns a +// body that contains a string. +// +// a.HTTPBodyContains(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPBodyContains(a.t, handler, method, url, values, str, msgAndArgs...) +} + +// HTTPBodyContainsf asserts that a specified handler returns a +// body that contains a string. +// +// a.HTTPBodyContainsf(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPBodyContainsf(a.t, handler, method, url, values, str, msg, args...) +} + +// HTTPBodyNotContains asserts that a specified handler returns a +// body that does not contain a string. +// +// a.HTTPBodyNotContains(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPBodyNotContains(a.t, handler, method, url, values, str, msgAndArgs...) +} + +// HTTPBodyNotContainsf asserts that a specified handler returns a +// body that does not contain a string. +// +// a.HTTPBodyNotContainsf(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPBodyNotContainsf(a.t, handler, method, url, values, str, msg, args...) +} + +// HTTPError asserts that a specified handler returns an error status code. +// +// a.HTTPError(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPError(a.t, handler, method, url, values, msgAndArgs...) +} + +// HTTPErrorf asserts that a specified handler returns an error status code. +// +// a.HTTPErrorf(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPErrorf(a.t, handler, method, url, values, msg, args...) +} + +// HTTPRedirect asserts that a specified handler returns a redirect status code. +// +// a.HTTPRedirect(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPRedirect(a.t, handler, method, url, values, msgAndArgs...) +} + +// HTTPRedirectf asserts that a specified handler returns a redirect status code. +// +// a.HTTPRedirectf(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPRedirectf(a.t, handler, method, url, values, msg, args...) +} + +// HTTPStatusCode asserts that a specified handler returns a specified status code. +// +// a.HTTPStatusCode(myHandler, "GET", "/notImplemented", nil, 501) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPStatusCode(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPStatusCode(a.t, handler, method, url, values, statuscode, msgAndArgs...) +} + +// HTTPStatusCodef asserts that a specified handler returns a specified status code. +// +// a.HTTPStatusCodef(myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPStatusCodef(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPStatusCodef(a.t, handler, method, url, values, statuscode, msg, args...) +} + +// HTTPSuccess asserts that a specified handler returns a success status code. +// +// a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPSuccess(a.t, handler, method, url, values, msgAndArgs...) +} + +// HTTPSuccessf asserts that a specified handler returns a success status code. +// +// a.HTTPSuccessf(myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPSuccessf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPSuccessf(a.t, handler, method, url, values, msg, args...) +} + +// Implements asserts that an object is implemented by the specified interface. +// +// a.Implements((*MyInterface)(nil), new(MyObject)) +func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Implements(a.t, interfaceObject, object, msgAndArgs...) +} + +// Implementsf asserts that an object is implemented by the specified interface. +// +// a.Implementsf((*MyInterface)(nil), new(MyObject), "error message %s", "formatted") +func (a *Assertions) Implementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Implementsf(a.t, interfaceObject, object, msg, args...) +} + +// InDelta asserts that the two numerals are within delta of each other. +// +// a.InDelta(math.Pi, 22/7.0, 0.01) +func (a *Assertions) InDelta(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InDelta(a.t, expected, actual, delta, msgAndArgs...) +} + +// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func (a *Assertions) InDeltaMapValues(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InDeltaMapValues(a.t, expected, actual, delta, msgAndArgs...) +} + +// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func (a *Assertions) InDeltaMapValuesf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InDeltaMapValuesf(a.t, expected, actual, delta, msg, args...) +} + +// InDeltaSlice is the same as InDelta, except it compares two slices. +func (a *Assertions) InDeltaSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InDeltaSlice(a.t, expected, actual, delta, msgAndArgs...) +} + +// InDeltaSlicef is the same as InDelta, except it compares two slices. +func (a *Assertions) InDeltaSlicef(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InDeltaSlicef(a.t, expected, actual, delta, msg, args...) +} + +// InDeltaf asserts that the two numerals are within delta of each other. +// +// a.InDeltaf(math.Pi, 22/7.0, 0.01, "error message %s", "formatted") +func (a *Assertions) InDeltaf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InDeltaf(a.t, expected, actual, delta, msg, args...) +} + +// InEpsilon asserts that expected and actual have a relative error less than epsilon +func (a *Assertions) InEpsilon(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InEpsilon(a.t, expected, actual, epsilon, msgAndArgs...) +} + +// InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. +func (a *Assertions) InEpsilonSlice(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InEpsilonSlice(a.t, expected, actual, epsilon, msgAndArgs...) +} + +// InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. +func (a *Assertions) InEpsilonSlicef(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InEpsilonSlicef(a.t, expected, actual, epsilon, msg, args...) +} + +// InEpsilonf asserts that expected and actual have a relative error less than epsilon +func (a *Assertions) InEpsilonf(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InEpsilonf(a.t, expected, actual, epsilon, msg, args...) +} + +// IsDecreasing asserts that the collection is decreasing +// +// a.IsDecreasing([]int{2, 1, 0}) +// a.IsDecreasing([]float{2, 1}) +// a.IsDecreasing([]string{"b", "a"}) +func (a *Assertions) IsDecreasing(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsDecreasing(a.t, object, msgAndArgs...) +} + +// IsDecreasingf asserts that the collection is decreasing +// +// a.IsDecreasingf([]int{2, 1, 0}, "error message %s", "formatted") +// a.IsDecreasingf([]float{2, 1}, "error message %s", "formatted") +// a.IsDecreasingf([]string{"b", "a"}, "error message %s", "formatted") +func (a *Assertions) IsDecreasingf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsDecreasingf(a.t, object, msg, args...) +} + +// IsIncreasing asserts that the collection is increasing +// +// a.IsIncreasing([]int{1, 2, 3}) +// a.IsIncreasing([]float{1, 2}) +// a.IsIncreasing([]string{"a", "b"}) +func (a *Assertions) IsIncreasing(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsIncreasing(a.t, object, msgAndArgs...) +} + +// IsIncreasingf asserts that the collection is increasing +// +// a.IsIncreasingf([]int{1, 2, 3}, "error message %s", "formatted") +// a.IsIncreasingf([]float{1, 2}, "error message %s", "formatted") +// a.IsIncreasingf([]string{"a", "b"}, "error message %s", "formatted") +func (a *Assertions) IsIncreasingf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsIncreasingf(a.t, object, msg, args...) +} + +// IsNonDecreasing asserts that the collection is not decreasing +// +// a.IsNonDecreasing([]int{1, 1, 2}) +// a.IsNonDecreasing([]float{1, 2}) +// a.IsNonDecreasing([]string{"a", "b"}) +func (a *Assertions) IsNonDecreasing(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsNonDecreasing(a.t, object, msgAndArgs...) +} + +// IsNonDecreasingf asserts that the collection is not decreasing +// +// a.IsNonDecreasingf([]int{1, 1, 2}, "error message %s", "formatted") +// a.IsNonDecreasingf([]float{1, 2}, "error message %s", "formatted") +// a.IsNonDecreasingf([]string{"a", "b"}, "error message %s", "formatted") +func (a *Assertions) IsNonDecreasingf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsNonDecreasingf(a.t, object, msg, args...) +} + +// IsNonIncreasing asserts that the collection is not increasing +// +// a.IsNonIncreasing([]int{2, 1, 1}) +// a.IsNonIncreasing([]float{2, 1}) +// a.IsNonIncreasing([]string{"b", "a"}) +func (a *Assertions) IsNonIncreasing(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsNonIncreasing(a.t, object, msgAndArgs...) +} + +// IsNonIncreasingf asserts that the collection is not increasing +// +// a.IsNonIncreasingf([]int{2, 1, 1}, "error message %s", "formatted") +// a.IsNonIncreasingf([]float{2, 1}, "error message %s", "formatted") +// a.IsNonIncreasingf([]string{"b", "a"}, "error message %s", "formatted") +func (a *Assertions) IsNonIncreasingf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsNonIncreasingf(a.t, object, msg, args...) +} + +// IsNotType asserts that the specified objects are not of the same type. +// +// a.IsNotType(&NotMyStruct{}, &MyStruct{}) +func (a *Assertions) IsNotType(theType interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsNotType(a.t, theType, object, msgAndArgs...) +} + +// IsNotTypef asserts that the specified objects are not of the same type. +// +// a.IsNotTypef(&NotMyStruct{}, &MyStruct{}, "error message %s", "formatted") +func (a *Assertions) IsNotTypef(theType interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsNotTypef(a.t, theType, object, msg, args...) +} + +// IsType asserts that the specified objects are of the same type. +// +// a.IsType(&MyStruct{}, &MyStruct{}) +func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsType(a.t, expectedType, object, msgAndArgs...) +} + +// IsTypef asserts that the specified objects are of the same type. +// +// a.IsTypef(&MyStruct{}, &MyStruct{}, "error message %s", "formatted") +func (a *Assertions) IsTypef(expectedType interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsTypef(a.t, expectedType, object, msg, args...) +} + +// JSONEq asserts that two JSON strings are equivalent. +// +// a.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) +func (a *Assertions) JSONEq(expected string, actual string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return JSONEq(a.t, expected, actual, msgAndArgs...) +} + +// JSONEqf asserts that two JSON strings are equivalent. +// +// a.JSONEqf(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") +func (a *Assertions) JSONEqf(expected string, actual string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return JSONEqf(a.t, expected, actual, msg, args...) +} + +// Len asserts that the specified object has specific length. +// Len also fails if the object has a type that len() not accept. +// +// a.Len(mySlice, 3) +func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Len(a.t, object, length, msgAndArgs...) +} + +// Lenf asserts that the specified object has specific length. +// Lenf also fails if the object has a type that len() not accept. +// +// a.Lenf(mySlice, 3, "error message %s", "formatted") +func (a *Assertions) Lenf(object interface{}, length int, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Lenf(a.t, object, length, msg, args...) +} + +// Less asserts that the first element is less than the second +// +// a.Less(1, 2) +// a.Less(float64(1), float64(2)) +// a.Less("a", "b") +func (a *Assertions) Less(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Less(a.t, e1, e2, msgAndArgs...) +} + +// LessOrEqual asserts that the first element is less than or equal to the second +// +// a.LessOrEqual(1, 2) +// a.LessOrEqual(2, 2) +// a.LessOrEqual("a", "b") +// a.LessOrEqual("b", "b") +func (a *Assertions) LessOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return LessOrEqual(a.t, e1, e2, msgAndArgs...) +} + +// LessOrEqualf asserts that the first element is less than or equal to the second +// +// a.LessOrEqualf(1, 2, "error message %s", "formatted") +// a.LessOrEqualf(2, 2, "error message %s", "formatted") +// a.LessOrEqualf("a", "b", "error message %s", "formatted") +// a.LessOrEqualf("b", "b", "error message %s", "formatted") +func (a *Assertions) LessOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return LessOrEqualf(a.t, e1, e2, msg, args...) +} + +// Lessf asserts that the first element is less than the second +// +// a.Lessf(1, 2, "error message %s", "formatted") +// a.Lessf(float64(1), float64(2), "error message %s", "formatted") +// a.Lessf("a", "b", "error message %s", "formatted") +func (a *Assertions) Lessf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Lessf(a.t, e1, e2, msg, args...) +} + +// Negative asserts that the specified element is negative +// +// a.Negative(-1) +// a.Negative(-1.23) +func (a *Assertions) Negative(e interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Negative(a.t, e, msgAndArgs...) +} + +// Negativef asserts that the specified element is negative +// +// a.Negativef(-1, "error message %s", "formatted") +// a.Negativef(-1.23, "error message %s", "formatted") +func (a *Assertions) Negativef(e interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Negativef(a.t, e, msg, args...) +} + +// Never asserts that the given condition doesn't satisfy in waitFor time, +// periodically checking the target function each tick. +// +// a.Never(func() bool { return false; }, time.Second, 10*time.Millisecond) +func (a *Assertions) Never(condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Never(a.t, condition, waitFor, tick, msgAndArgs...) +} + +// Neverf asserts that the given condition doesn't satisfy in waitFor time, +// periodically checking the target function each tick. +// +// a.Neverf(func() bool { return false; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func (a *Assertions) Neverf(condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Neverf(a.t, condition, waitFor, tick, msg, args...) +} + +// Nil asserts that the specified object is nil. +// +// a.Nil(err) +func (a *Assertions) Nil(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Nil(a.t, object, msgAndArgs...) +} + +// Nilf asserts that the specified object is nil. +// +// a.Nilf(err, "error message %s", "formatted") +func (a *Assertions) Nilf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Nilf(a.t, object, msg, args...) +} + +// NoDirExists checks whether a directory does not exist in the given path. +// It fails if the path points to an existing _directory_ only. +func (a *Assertions) NoDirExists(path string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NoDirExists(a.t, path, msgAndArgs...) +} + +// NoDirExistsf checks whether a directory does not exist in the given path. +// It fails if the path points to an existing _directory_ only. +func (a *Assertions) NoDirExistsf(path string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NoDirExistsf(a.t, path, msg, args...) +} + +// NoError asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if a.NoError(err) { +// assert.Equal(t, expectedObj, actualObj) +// } +func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NoError(a.t, err, msgAndArgs...) +} + +// NoErrorf asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if a.NoErrorf(err, "error message %s", "formatted") { +// assert.Equal(t, expectedObj, actualObj) +// } +func (a *Assertions) NoErrorf(err error, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NoErrorf(a.t, err, msg, args...) +} + +// NoFileExists checks whether a file does not exist in a given path. It fails +// if the path points to an existing _file_ only. +func (a *Assertions) NoFileExists(path string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NoFileExists(a.t, path, msgAndArgs...) +} + +// NoFileExistsf checks whether a file does not exist in a given path. It fails +// if the path points to an existing _file_ only. +func (a *Assertions) NoFileExistsf(path string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NoFileExistsf(a.t, path, msg, args...) +} + +// NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// a.NotContains("Hello World", "Earth") +// a.NotContains(["Hello", "World"], "Earth") +// a.NotContains({"Hello": "World"}, "Earth") +func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotContains(a.t, s, contains, msgAndArgs...) +} + +// NotContainsf asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// a.NotContainsf("Hello World", "Earth", "error message %s", "formatted") +// a.NotContainsf(["Hello", "World"], "Earth", "error message %s", "formatted") +// a.NotContainsf({"Hello": "World"}, "Earth", "error message %s", "formatted") +func (a *Assertions) NotContainsf(s interface{}, contains interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotContainsf(a.t, s, contains, msg, args...) +} + +// NotElementsMatch asserts that the specified listA(array, slice...) is NOT equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should not match. +// This is an inverse of ElementsMatch. +// +// a.NotElementsMatch([1, 1, 2, 3], [1, 1, 2, 3]) -> false +// +// a.NotElementsMatch([1, 1, 2, 3], [1, 2, 3]) -> true +// +// a.NotElementsMatch([1, 2, 3], [1, 2, 4]) -> true +func (a *Assertions) NotElementsMatch(listA interface{}, listB interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotElementsMatch(a.t, listA, listB, msgAndArgs...) +} + +// NotElementsMatchf asserts that the specified listA(array, slice...) is NOT equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should not match. +// This is an inverse of ElementsMatch. +// +// a.NotElementsMatchf([1, 1, 2, 3], [1, 1, 2, 3], "error message %s", "formatted") -> false +// +// a.NotElementsMatchf([1, 1, 2, 3], [1, 2, 3], "error message %s", "formatted") -> true +// +// a.NotElementsMatchf([1, 2, 3], [1, 2, 4], "error message %s", "formatted") -> true +func (a *Assertions) NotElementsMatchf(listA interface{}, listB interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotElementsMatchf(a.t, listA, listB, msg, args...) +} + +// NotEmpty asserts that the specified object is NOT [Empty]. +// +// if a.NotEmpty(obj) { +// assert.Equal(t, "two", obj[1]) +// } +func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotEmpty(a.t, object, msgAndArgs...) +} + +// NotEmptyf asserts that the specified object is NOT [Empty]. +// +// if a.NotEmptyf(obj, "error message %s", "formatted") { +// assert.Equal(t, "two", obj[1]) +// } +func (a *Assertions) NotEmptyf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotEmptyf(a.t, object, msg, args...) +} + +// NotEqual asserts that the specified values are NOT equal. +// +// a.NotEqual(obj1, obj2) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). +func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotEqual(a.t, expected, actual, msgAndArgs...) +} + +// NotEqualValues asserts that two objects are not equal even when converted to the same type +// +// a.NotEqualValues(obj1, obj2) +func (a *Assertions) NotEqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotEqualValues(a.t, expected, actual, msgAndArgs...) +} + +// NotEqualValuesf asserts that two objects are not equal even when converted to the same type +// +// a.NotEqualValuesf(obj1, obj2, "error message %s", "formatted") +func (a *Assertions) NotEqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotEqualValuesf(a.t, expected, actual, msg, args...) +} + +// NotEqualf asserts that the specified values are NOT equal. +// +// a.NotEqualf(obj1, obj2, "error message %s", "formatted") +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). +func (a *Assertions) NotEqualf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotEqualf(a.t, expected, actual, msg, args...) +} + +// NotErrorAs asserts that none of the errors in err's chain matches target, +// but if so, sets target to that error value. +func (a *Assertions) NotErrorAs(err error, target interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotErrorAs(a.t, err, target, msgAndArgs...) +} + +// NotErrorAsf asserts that none of the errors in err's chain matches target, +// but if so, sets target to that error value. +func (a *Assertions) NotErrorAsf(err error, target interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotErrorAsf(a.t, err, target, msg, args...) +} + +// NotErrorIs asserts that none of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func (a *Assertions) NotErrorIs(err error, target error, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotErrorIs(a.t, err, target, msgAndArgs...) +} + +// NotErrorIsf asserts that none of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func (a *Assertions) NotErrorIsf(err error, target error, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotErrorIsf(a.t, err, target, msg, args...) +} + +// NotImplements asserts that an object does not implement the specified interface. +// +// a.NotImplements((*MyInterface)(nil), new(MyObject)) +func (a *Assertions) NotImplements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotImplements(a.t, interfaceObject, object, msgAndArgs...) +} + +// NotImplementsf asserts that an object does not implement the specified interface. +// +// a.NotImplementsf((*MyInterface)(nil), new(MyObject), "error message %s", "formatted") +func (a *Assertions) NotImplementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotImplementsf(a.t, interfaceObject, object, msg, args...) +} + +// NotNil asserts that the specified object is not nil. +// +// a.NotNil(err) +func (a *Assertions) NotNil(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotNil(a.t, object, msgAndArgs...) +} + +// NotNilf asserts that the specified object is not nil. +// +// a.NotNilf(err, "error message %s", "formatted") +func (a *Assertions) NotNilf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotNilf(a.t, object, msg, args...) +} + +// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// a.NotPanics(func(){ RemainCalm() }) +func (a *Assertions) NotPanics(f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotPanics(a.t, f, msgAndArgs...) +} + +// NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// a.NotPanicsf(func(){ RemainCalm() }, "error message %s", "formatted") +func (a *Assertions) NotPanicsf(f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotPanicsf(a.t, f, msg, args...) +} + +// NotRegexp asserts that a specified regexp does not match a string. +// +// a.NotRegexp(regexp.MustCompile("starts"), "it's starting") +// a.NotRegexp("^start", "it's not starting") +func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotRegexp(a.t, rx, str, msgAndArgs...) +} + +// NotRegexpf asserts that a specified regexp does not match a string. +// +// a.NotRegexpf(regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") +// a.NotRegexpf("^start", "it's not starting", "error message %s", "formatted") +func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotRegexpf(a.t, rx, str, msg, args...) +} + +// NotSame asserts that two pointers do not reference the same object. +// +// a.NotSame(ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) NotSame(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotSame(a.t, expected, actual, msgAndArgs...) +} + +// NotSamef asserts that two pointers do not reference the same object. +// +// a.NotSamef(ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) NotSamef(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotSamef(a.t, expected, actual, msg, args...) +} + +// NotSubset asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// a.NotSubset([1, 3, 4], [1, 2]) +// a.NotSubset({"x": 1, "y": 2}, {"z": 3}) +// a.NotSubset([1, 3, 4], {1: "one", 2: "two"}) +// a.NotSubset({"x": 1, "y": 2}, ["z"]) +func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotSubset(a.t, list, subset, msgAndArgs...) +} + +// NotSubsetf asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// a.NotSubsetf([1, 3, 4], [1, 2], "error message %s", "formatted") +// a.NotSubsetf({"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") +// a.NotSubsetf([1, 3, 4], {1: "one", 2: "two"}, "error message %s", "formatted") +// a.NotSubsetf({"x": 1, "y": 2}, ["z"], "error message %s", "formatted") +func (a *Assertions) NotSubsetf(list interface{}, subset interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotSubsetf(a.t, list, subset, msg, args...) +} + +// NotZero asserts that i is not the zero value for its type. +func (a *Assertions) NotZero(i interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotZero(a.t, i, msgAndArgs...) +} + +// NotZerof asserts that i is not the zero value for its type. +func (a *Assertions) NotZerof(i interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotZerof(a.t, i, msg, args...) +} + +// Panics asserts that the code inside the specified PanicTestFunc panics. +// +// a.Panics(func(){ GoCrazy() }) +func (a *Assertions) Panics(f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Panics(a.t, f, msgAndArgs...) +} + +// PanicsWithError asserts that the code inside the specified PanicTestFunc +// panics, and that the recovered panic value is an error that satisfies the +// EqualError comparison. +// +// a.PanicsWithError("crazy error", func(){ GoCrazy() }) +func (a *Assertions) PanicsWithError(errString string, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return PanicsWithError(a.t, errString, f, msgAndArgs...) +} + +// PanicsWithErrorf asserts that the code inside the specified PanicTestFunc +// panics, and that the recovered panic value is an error that satisfies the +// EqualError comparison. +// +// a.PanicsWithErrorf("crazy error", func(){ GoCrazy() }, "error message %s", "formatted") +func (a *Assertions) PanicsWithErrorf(errString string, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return PanicsWithErrorf(a.t, errString, f, msg, args...) +} + +// PanicsWithValue asserts that the code inside the specified PanicTestFunc panics, and that +// the recovered panic value equals the expected panic value. +// +// a.PanicsWithValue("crazy error", func(){ GoCrazy() }) +func (a *Assertions) PanicsWithValue(expected interface{}, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return PanicsWithValue(a.t, expected, f, msgAndArgs...) +} + +// PanicsWithValuef asserts that the code inside the specified PanicTestFunc panics, and that +// the recovered panic value equals the expected panic value. +// +// a.PanicsWithValuef("crazy error", func(){ GoCrazy() }, "error message %s", "formatted") +func (a *Assertions) PanicsWithValuef(expected interface{}, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return PanicsWithValuef(a.t, expected, f, msg, args...) +} + +// Panicsf asserts that the code inside the specified PanicTestFunc panics. +// +// a.Panicsf(func(){ GoCrazy() }, "error message %s", "formatted") +func (a *Assertions) Panicsf(f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Panicsf(a.t, f, msg, args...) +} + +// Positive asserts that the specified element is positive +// +// a.Positive(1) +// a.Positive(1.23) +func (a *Assertions) Positive(e interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Positive(a.t, e, msgAndArgs...) +} + +// Positivef asserts that the specified element is positive +// +// a.Positivef(1, "error message %s", "formatted") +// a.Positivef(1.23, "error message %s", "formatted") +func (a *Assertions) Positivef(e interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Positivef(a.t, e, msg, args...) +} + +// Regexp asserts that a specified regexp matches a string. +// +// a.Regexp(regexp.MustCompile("start"), "it's starting") +// a.Regexp("start...$", "it's not starting") +func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Regexp(a.t, rx, str, msgAndArgs...) +} + +// Regexpf asserts that a specified regexp matches a string. +// +// a.Regexpf(regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") +// a.Regexpf("start...$", "it's not starting", "error message %s", "formatted") +func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Regexpf(a.t, rx, str, msg, args...) +} + +// Same asserts that two pointers reference the same object. +// +// a.Same(ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) Same(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Same(a.t, expected, actual, msgAndArgs...) +} + +// Samef asserts that two pointers reference the same object. +// +// a.Samef(ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) Samef(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Samef(a.t, expected, actual, msg, args...) +} + +// Subset asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// a.Subset([1, 2, 3], [1, 2]) +// a.Subset({"x": 1, "y": 2}, {"x": 1}) +// a.Subset([1, 2, 3], {1: "one", 2: "two"}) +// a.Subset({"x": 1, "y": 2}, ["x"]) +func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Subset(a.t, list, subset, msgAndArgs...) +} + +// Subsetf asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// a.Subsetf([1, 2, 3], [1, 2], "error message %s", "formatted") +// a.Subsetf({"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") +// a.Subsetf([1, 2, 3], {1: "one", 2: "two"}, "error message %s", "formatted") +// a.Subsetf({"x": 1, "y": 2}, ["x"], "error message %s", "formatted") +func (a *Assertions) Subsetf(list interface{}, subset interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Subsetf(a.t, list, subset, msg, args...) +} + +// True asserts that the specified value is true. +// +// a.True(myBool) +func (a *Assertions) True(value bool, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return True(a.t, value, msgAndArgs...) +} + +// Truef asserts that the specified value is true. +// +// a.Truef(myBool, "error message %s", "formatted") +func (a *Assertions) Truef(value bool, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Truef(a.t, value, msg, args...) +} + +// WithinDuration asserts that the two times are within duration delta of each other. +// +// a.WithinDuration(time.Now(), time.Now(), 10*time.Second) +func (a *Assertions) WithinDuration(expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return WithinDuration(a.t, expected, actual, delta, msgAndArgs...) +} + +// WithinDurationf asserts that the two times are within duration delta of each other. +// +// a.WithinDurationf(time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") +func (a *Assertions) WithinDurationf(expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return WithinDurationf(a.t, expected, actual, delta, msg, args...) +} + +// WithinRange asserts that a time is within a time range (inclusive). +// +// a.WithinRange(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) +func (a *Assertions) WithinRange(actual time.Time, start time.Time, end time.Time, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return WithinRange(a.t, actual, start, end, msgAndArgs...) +} + +// WithinRangef asserts that a time is within a time range (inclusive). +// +// a.WithinRangef(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") +func (a *Assertions) WithinRangef(actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return WithinRangef(a.t, actual, start, end, msg, args...) +} + +// YAMLEq asserts that two YAML strings are equivalent. +func (a *Assertions) YAMLEq(expected string, actual string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return YAMLEq(a.t, expected, actual, msgAndArgs...) +} + +// YAMLEqf asserts that two YAML strings are equivalent. +func (a *Assertions) YAMLEqf(expected string, actual string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return YAMLEqf(a.t, expected, actual, msg, args...) +} + +// Zero asserts that i is the zero value for its type. +func (a *Assertions) Zero(i interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Zero(a.t, i, msgAndArgs...) +} + +// Zerof asserts that i is the zero value for its type. +func (a *Assertions) Zerof(i interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return Zerof(a.t, i, msg, args...) +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl new file mode 100644 index 00000000..188bb9e1 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl @@ -0,0 +1,5 @@ +{{.CommentWithoutT "a"}} +func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) bool { + if h, ok := a.t.(tHelper); ok { h.Helper() } + return {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_order.go b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_order.go new file mode 100644 index 00000000..2fdf80fd --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/assertion_order.go @@ -0,0 +1,81 @@ +package assert + +import ( + "fmt" + "reflect" +) + +// isOrdered checks that collection contains orderable elements. +func isOrdered(t TestingT, object interface{}, allowedComparesResults []compareResult, failMessage string, msgAndArgs ...interface{}) bool { + objKind := reflect.TypeOf(object).Kind() + if objKind != reflect.Slice && objKind != reflect.Array { + return false + } + + objValue := reflect.ValueOf(object) + objLen := objValue.Len() + + if objLen <= 1 { + return true + } + + value := objValue.Index(0) + valueInterface := value.Interface() + firstValueKind := value.Kind() + + for i := 1; i < objLen; i++ { + prevValue := value + prevValueInterface := valueInterface + + value = objValue.Index(i) + valueInterface = value.Interface() + + compareResult, isComparable := compare(prevValueInterface, valueInterface, firstValueKind) + + if !isComparable { + return Fail(t, fmt.Sprintf(`Can not compare type "%T" and "%T"`, value, prevValue), msgAndArgs...) + } + + if !containsValue(allowedComparesResults, compareResult) { + return Fail(t, fmt.Sprintf(failMessage, prevValue, value), msgAndArgs...) + } + } + + return true +} + +// IsIncreasing asserts that the collection is increasing +// +// assert.IsIncreasing(t, []int{1, 2, 3}) +// assert.IsIncreasing(t, []float{1, 2}) +// assert.IsIncreasing(t, []string{"a", "b"}) +func IsIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + return isOrdered(t, object, []compareResult{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs...) +} + +// IsNonIncreasing asserts that the collection is not increasing +// +// assert.IsNonIncreasing(t, []int{2, 1, 1}) +// assert.IsNonIncreasing(t, []float{2, 1}) +// assert.IsNonIncreasing(t, []string{"b", "a"}) +func IsNonIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + return isOrdered(t, object, []compareResult{compareEqual, compareGreater}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs...) +} + +// IsDecreasing asserts that the collection is decreasing +// +// assert.IsDecreasing(t, []int{2, 1, 0}) +// assert.IsDecreasing(t, []float{2, 1}) +// assert.IsDecreasing(t, []string{"b", "a"}) +func IsDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + return isOrdered(t, object, []compareResult{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs...) +} + +// IsNonDecreasing asserts that the collection is not decreasing +// +// assert.IsNonDecreasing(t, []int{1, 1, 2}) +// assert.IsNonDecreasing(t, []float{1, 2}) +// assert.IsNonDecreasing(t, []string{"a", "b"}) +func IsNonDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + return isOrdered(t, object, []compareResult{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs...) +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/assertions.go b/scraper-go/vendor/github.com/stretchr/testify/assert/assertions.go new file mode 100644 index 00000000..de8de0cb --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/assertions.go @@ -0,0 +1,2295 @@ +package assert + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "reflect" + "regexp" + "runtime" + "runtime/debug" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/davecgh/go-spew/spew" + "github.com/pmezard/go-difflib/difflib" + + // Wrapper around gopkg.in/yaml.v3 + "github.com/stretchr/testify/assert/yaml" +) + +//go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=assert -template=assertion_format.go.tmpl" + +// TestingT is an interface wrapper around *testing.T +type TestingT interface { + Errorf(format string, args ...interface{}) +} + +// ComparisonAssertionFunc is a common function prototype when comparing two values. Can be useful +// for table driven tests. +type ComparisonAssertionFunc func(TestingT, interface{}, interface{}, ...interface{}) bool + +// ValueAssertionFunc is a common function prototype when validating a single value. Can be useful +// for table driven tests. +type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) bool + +// BoolAssertionFunc is a common function prototype when validating a bool value. Can be useful +// for table driven tests. +type BoolAssertionFunc func(TestingT, bool, ...interface{}) bool + +// ErrorAssertionFunc is a common function prototype when validating an error value. Can be useful +// for table driven tests. +type ErrorAssertionFunc func(TestingT, error, ...interface{}) bool + +// PanicAssertionFunc is a common function prototype when validating a panic value. Can be useful +// for table driven tests. +type PanicAssertionFunc = func(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool + +// Comparison is a custom function that returns true on success and false on failure +type Comparison func() (success bool) + +/* + Helper functions +*/ + +// ObjectsAreEqual determines if two objects are considered equal. +// +// This function does no assertion of any kind. +func ObjectsAreEqual(expected, actual interface{}) bool { + if expected == nil || actual == nil { + return expected == actual + } + + exp, ok := expected.([]byte) + if !ok { + return reflect.DeepEqual(expected, actual) + } + + act, ok := actual.([]byte) + if !ok { + return false + } + if exp == nil || act == nil { + return exp == nil && act == nil + } + return bytes.Equal(exp, act) +} + +// copyExportedFields iterates downward through nested data structures and creates a copy +// that only contains the exported struct fields. +func copyExportedFields(expected interface{}) interface{} { + if isNil(expected) { + return expected + } + + expectedType := reflect.TypeOf(expected) + expectedKind := expectedType.Kind() + expectedValue := reflect.ValueOf(expected) + + switch expectedKind { + case reflect.Struct: + result := reflect.New(expectedType).Elem() + for i := 0; i < expectedType.NumField(); i++ { + field := expectedType.Field(i) + isExported := field.IsExported() + if isExported { + fieldValue := expectedValue.Field(i) + if isNil(fieldValue) || isNil(fieldValue.Interface()) { + continue + } + newValue := copyExportedFields(fieldValue.Interface()) + result.Field(i).Set(reflect.ValueOf(newValue)) + } + } + return result.Interface() + + case reflect.Ptr: + result := reflect.New(expectedType.Elem()) + unexportedRemoved := copyExportedFields(expectedValue.Elem().Interface()) + result.Elem().Set(reflect.ValueOf(unexportedRemoved)) + return result.Interface() + + case reflect.Array, reflect.Slice: + var result reflect.Value + if expectedKind == reflect.Array { + result = reflect.New(reflect.ArrayOf(expectedValue.Len(), expectedType.Elem())).Elem() + } else { + result = reflect.MakeSlice(expectedType, expectedValue.Len(), expectedValue.Len()) + } + for i := 0; i < expectedValue.Len(); i++ { + index := expectedValue.Index(i) + if isNil(index) { + continue + } + unexportedRemoved := copyExportedFields(index.Interface()) + result.Index(i).Set(reflect.ValueOf(unexportedRemoved)) + } + return result.Interface() + + case reflect.Map: + result := reflect.MakeMap(expectedType) + for _, k := range expectedValue.MapKeys() { + index := expectedValue.MapIndex(k) + unexportedRemoved := copyExportedFields(index.Interface()) + result.SetMapIndex(k, reflect.ValueOf(unexportedRemoved)) + } + return result.Interface() + + default: + return expected + } +} + +// ObjectsExportedFieldsAreEqual determines if the exported (public) fields of two objects are +// considered equal. This comparison of only exported fields is applied recursively to nested data +// structures. +// +// This function does no assertion of any kind. +// +// Deprecated: Use [EqualExportedValues] instead. +func ObjectsExportedFieldsAreEqual(expected, actual interface{}) bool { + expectedCleaned := copyExportedFields(expected) + actualCleaned := copyExportedFields(actual) + return ObjectsAreEqualValues(expectedCleaned, actualCleaned) +} + +// ObjectsAreEqualValues gets whether two objects are equal, or if their +// values are equal. +func ObjectsAreEqualValues(expected, actual interface{}) bool { + if ObjectsAreEqual(expected, actual) { + return true + } + + expectedValue := reflect.ValueOf(expected) + actualValue := reflect.ValueOf(actual) + if !expectedValue.IsValid() || !actualValue.IsValid() { + return false + } + + expectedType := expectedValue.Type() + actualType := actualValue.Type() + if !expectedType.ConvertibleTo(actualType) { + return false + } + + if !isNumericType(expectedType) || !isNumericType(actualType) { + // Attempt comparison after type conversion + return reflect.DeepEqual( + expectedValue.Convert(actualType).Interface(), actual, + ) + } + + // If BOTH values are numeric, there are chances of false positives due + // to overflow or underflow. So, we need to make sure to always convert + // the smaller type to a larger type before comparing. + if expectedType.Size() >= actualType.Size() { + return actualValue.Convert(expectedType).Interface() == expected + } + + return expectedValue.Convert(actualType).Interface() == actual +} + +// isNumericType returns true if the type is one of: +// int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, +// float32, float64, complex64, complex128 +func isNumericType(t reflect.Type) bool { + return t.Kind() >= reflect.Int && t.Kind() <= reflect.Complex128 +} + +/* CallerInfo is necessary because the assert functions use the testing object +internally, causing it to print the file:line of the assert method, rather than where +the problem actually occurred in calling code.*/ + +// CallerInfo returns an array of strings containing the file and line number +// of each stack frame leading from the current test to the assert call that +// failed. +func CallerInfo() []string { + var pc uintptr + var file string + var line int + var name string + + const stackFrameBufferSize = 10 + pcs := make([]uintptr, stackFrameBufferSize) + + callers := []string{} + offset := 1 + + for { + n := runtime.Callers(offset, pcs) + + if n == 0 { + break + } + + frames := runtime.CallersFrames(pcs[:n]) + + for { + frame, more := frames.Next() + pc = frame.PC + file = frame.File + line = frame.Line + + // This is a huge edge case, but it will panic if this is the case, see #180 + if file == "" { + break + } + + f := runtime.FuncForPC(pc) + if f == nil { + break + } + name = f.Name() + + // testing.tRunner is the standard library function that calls + // tests. Subtests are called directly by tRunner, without going through + // the Test/Benchmark/Example function that contains the t.Run calls, so + // with subtests we should break when we hit tRunner, without adding it + // to the list of callers. + if name == "testing.tRunner" { + break + } + + parts := strings.Split(file, "/") + if len(parts) > 1 { + filename := parts[len(parts)-1] + dir := parts[len(parts)-2] + if (dir != "assert" && dir != "mock" && dir != "require") || filename == "mock_test.go" { + callers = append(callers, fmt.Sprintf("%s:%d", file, line)) + } + } + + // Drop the package + dotPos := strings.LastIndexByte(name, '.') + name = name[dotPos+1:] + if isTest(name, "Test") || + isTest(name, "Benchmark") || + isTest(name, "Example") { + break + } + + if !more { + break + } + } + + // Next batch + offset += cap(pcs) + } + + return callers +} + +// Stolen from the `go test` tool. +// isTest tells whether name looks like a test (or benchmark, according to prefix). +// It is a Test (say) if there is a character after Test that is not a lower-case letter. +// We don't want TesticularCancer. +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { // "Test" is ok + return true + } + r, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(r) +} + +func messageFromMsgAndArgs(msgAndArgs ...interface{}) string { + if len(msgAndArgs) == 0 || msgAndArgs == nil { + return "" + } + if len(msgAndArgs) == 1 { + msg := msgAndArgs[0] + if msgAsStr, ok := msg.(string); ok { + return msgAsStr + } + return fmt.Sprintf("%+v", msg) + } + if len(msgAndArgs) > 1 { + return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + } + return "" +} + +// Aligns the provided message so that all lines after the first line start at the same location as the first line. +// Assumes that the first line starts at the correct location (after carriage return, tab, label, spacer and tab). +// The longestLabelLen parameter specifies the length of the longest label in the output (required because this is the +// basis on which the alignment occurs). +func indentMessageLines(message string, longestLabelLen int) string { + outBuf := new(bytes.Buffer) + + for i, scanner := 0, bufio.NewScanner(strings.NewReader(message)); scanner.Scan(); i++ { + // no need to align first line because it starts at the correct location (after the label) + if i != 0 { + // append alignLen+1 spaces to align with "{{longestLabel}}:" before adding tab + outBuf.WriteString("\n\t" + strings.Repeat(" ", longestLabelLen+1) + "\t") + } + outBuf.WriteString(scanner.Text()) + } + + return outBuf.String() +} + +type failNower interface { + FailNow() +} + +// FailNow fails test +func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + Fail(t, failureMessage, msgAndArgs...) + + // We cannot extend TestingT with FailNow() and + // maintain backwards compatibility, so we fallback + // to panicking when FailNow is not available in + // TestingT. + // See issue #263 + + if t, ok := t.(failNower); ok { + t.FailNow() + } else { + panic("test failed and t is missing `FailNow()`") + } + return false +} + +// Fail reports a failure through +func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + content := []labeledContent{ + {"Error Trace", strings.Join(CallerInfo(), "\n\t\t\t")}, + {"Error", failureMessage}, + } + + // Add test name if the Go version supports it + if n, ok := t.(interface { + Name() string + }); ok { + content = append(content, labeledContent{"Test", n.Name()}) + } + + message := messageFromMsgAndArgs(msgAndArgs...) + if len(message) > 0 { + content = append(content, labeledContent{"Messages", message}) + } + + t.Errorf("\n%s", ""+labeledOutput(content...)) + + return false +} + +type labeledContent struct { + label string + content string +} + +// labeledOutput returns a string consisting of the provided labeledContent. Each labeled output is appended in the following manner: +// +// \t{{label}}:{{align_spaces}}\t{{content}}\n +// +// The initial carriage return is required to undo/erase any padding added by testing.T.Errorf. The "\t{{label}}:" is for the label. +// If a label is shorter than the longest label provided, padding spaces are added to make all the labels match in length. Once this +// alignment is achieved, "\t{{content}}\n" is added for the output. +// +// If the content of the labeledOutput contains line breaks, the subsequent lines are aligned so that they start at the same location as the first line. +func labeledOutput(content ...labeledContent) string { + longestLabel := 0 + for _, v := range content { + if len(v.label) > longestLabel { + longestLabel = len(v.label) + } + } + var output string + for _, v := range content { + output += "\t" + v.label + ":" + strings.Repeat(" ", longestLabel-len(v.label)) + "\t" + indentMessageLines(v.content, longestLabel) + "\n" + } + return output +} + +// Implements asserts that an object is implemented by the specified interface. +// +// assert.Implements(t, (*MyInterface)(nil), new(MyObject)) +func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + interfaceType := reflect.TypeOf(interfaceObject).Elem() + + if object == nil { + return Fail(t, fmt.Sprintf("Cannot check if nil implements %v", interfaceType), msgAndArgs...) + } + if !reflect.TypeOf(object).Implements(interfaceType) { + return Fail(t, fmt.Sprintf("%T must implement %v", object, interfaceType), msgAndArgs...) + } + + return true +} + +// NotImplements asserts that an object does not implement the specified interface. +// +// assert.NotImplements(t, (*MyInterface)(nil), new(MyObject)) +func NotImplements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + interfaceType := reflect.TypeOf(interfaceObject).Elem() + + if object == nil { + return Fail(t, fmt.Sprintf("Cannot check if nil does not implement %v", interfaceType), msgAndArgs...) + } + if reflect.TypeOf(object).Implements(interfaceType) { + return Fail(t, fmt.Sprintf("%T implements %v", object, interfaceType), msgAndArgs...) + } + + return true +} + +func isType(expectedType, object interface{}) bool { + return ObjectsAreEqual(reflect.TypeOf(object), reflect.TypeOf(expectedType)) +} + +// IsType asserts that the specified objects are of the same type. +// +// assert.IsType(t, &MyStruct{}, &MyStruct{}) +func IsType(t TestingT, expectedType, object interface{}, msgAndArgs ...interface{}) bool { + if isType(expectedType, object) { + return true + } + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, fmt.Sprintf("Object expected to be of type %T, but was %T", expectedType, object), msgAndArgs...) +} + +// IsNotType asserts that the specified objects are not of the same type. +// +// assert.IsNotType(t, &NotMyStruct{}, &MyStruct{}) +func IsNotType(t TestingT, theType, object interface{}, msgAndArgs ...interface{}) bool { + if !isType(theType, object) { + return true + } + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, fmt.Sprintf("Object type expected to be different than %T", theType), msgAndArgs...) +} + +// Equal asserts that two objects are equal. +// +// assert.Equal(t, 123, 123) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if err := validateEqualArgs(expected, actual); err != nil { + return Fail(t, fmt.Sprintf("Invalid operation: %#v == %#v (%s)", + expected, actual, err), msgAndArgs...) + } + + if !ObjectsAreEqual(expected, actual) { + diff := diff(expected, actual) + expected, actual = formatUnequalValues(expected, actual) + return Fail(t, fmt.Sprintf("Not equal: \n"+ + "expected: %s\n"+ + "actual : %s%s", expected, actual, diff), msgAndArgs...) + } + + return true +} + +// validateEqualArgs checks whether provided arguments can be safely used in the +// Equal/NotEqual functions. +func validateEqualArgs(expected, actual interface{}) error { + if expected == nil && actual == nil { + return nil + } + + if isFunction(expected) || isFunction(actual) { + return errors.New("cannot take func type as argument") + } + return nil +} + +// Same asserts that two pointers reference the same object. +// +// assert.Same(t, ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func Same(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + same, ok := samePointers(expected, actual) + if !ok { + return Fail(t, "Both arguments must be pointers", msgAndArgs...) + } + + if !same { + // both are pointers but not the same type & pointing to the same address + return Fail(t, fmt.Sprintf("Not same: \n"+ + "expected: %p %#[1]v\n"+ + "actual : %p %#[2]v", + expected, actual), msgAndArgs...) + } + + return true +} + +// NotSame asserts that two pointers do not reference the same object. +// +// assert.NotSame(t, ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func NotSame(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + same, ok := samePointers(expected, actual) + if !ok { + // fails when the arguments are not pointers + return !(Fail(t, "Both arguments must be pointers", msgAndArgs...)) + } + + if same { + return Fail(t, fmt.Sprintf( + "Expected and actual point to the same object: %p %#[1]v", + expected), msgAndArgs...) + } + return true +} + +// samePointers checks if two generic interface objects are pointers of the same +// type pointing to the same object. It returns two values: same indicating if +// they are the same type and point to the same object, and ok indicating that +// both inputs are pointers. +func samePointers(first, second interface{}) (same bool, ok bool) { + firstPtr, secondPtr := reflect.ValueOf(first), reflect.ValueOf(second) + if firstPtr.Kind() != reflect.Ptr || secondPtr.Kind() != reflect.Ptr { + return false, false // not both are pointers + } + + firstType, secondType := reflect.TypeOf(first), reflect.TypeOf(second) + if firstType != secondType { + return false, true // both are pointers, but of different types + } + + // compare pointer addresses + return first == second, true +} + +// formatUnequalValues takes two values of arbitrary types and returns string +// representations appropriate to be presented to the user. +// +// If the values are not of like type, the returned strings will be prefixed +// with the type name, and the value will be enclosed in parentheses similar +// to a type conversion in the Go grammar. +func formatUnequalValues(expected, actual interface{}) (e string, a string) { + if reflect.TypeOf(expected) != reflect.TypeOf(actual) { + return fmt.Sprintf("%T(%s)", expected, truncatingFormat(expected)), + fmt.Sprintf("%T(%s)", actual, truncatingFormat(actual)) + } + switch expected.(type) { + case time.Duration: + return fmt.Sprintf("%v", expected), fmt.Sprintf("%v", actual) + } + return truncatingFormat(expected), truncatingFormat(actual) +} + +// truncatingFormat formats the data and truncates it if it's too long. +// +// This helps keep formatted error messages lines from exceeding the +// bufio.MaxScanTokenSize max line length that the go testing framework imposes. +func truncatingFormat(data interface{}) string { + value := fmt.Sprintf("%#v", data) + max := bufio.MaxScanTokenSize - 100 // Give us some space the type info too if needed. + if len(value) > max { + value = value[0:max] + "<... truncated>" + } + return value +} + +// EqualValues asserts that two objects are equal or convertible to the larger +// type and equal. +// +// assert.EqualValues(t, uint32(123), int32(123)) +func EqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + if !ObjectsAreEqualValues(expected, actual) { + diff := diff(expected, actual) + expected, actual = formatUnequalValues(expected, actual) + return Fail(t, fmt.Sprintf("Not equal: \n"+ + "expected: %s\n"+ + "actual : %s%s", expected, actual, diff), msgAndArgs...) + } + + return true +} + +// EqualExportedValues asserts that the types of two objects are equal and their public +// fields are also equal. This is useful for comparing structs that have private fields +// that could potentially differ. +// +// type S struct { +// Exported int +// notExported int +// } +// assert.EqualExportedValues(t, S{1, 2}, S{1, 3}) => true +// assert.EqualExportedValues(t, S{1, 2}, S{2, 3}) => false +func EqualExportedValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + aType := reflect.TypeOf(expected) + bType := reflect.TypeOf(actual) + + if aType != bType { + return Fail(t, fmt.Sprintf("Types expected to match exactly\n\t%v != %v", aType, bType), msgAndArgs...) + } + + expected = copyExportedFields(expected) + actual = copyExportedFields(actual) + + if !ObjectsAreEqualValues(expected, actual) { + diff := diff(expected, actual) + expected, actual = formatUnequalValues(expected, actual) + return Fail(t, fmt.Sprintf("Not equal (comparing only exported fields): \n"+ + "expected: %s\n"+ + "actual : %s%s", expected, actual, diff), msgAndArgs...) + } + + return true +} + +// Exactly asserts that two objects are equal in value and type. +// +// assert.Exactly(t, int32(123), int64(123)) +func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + aType := reflect.TypeOf(expected) + bType := reflect.TypeOf(actual) + + if aType != bType { + return Fail(t, fmt.Sprintf("Types expected to match exactly\n\t%v != %v", aType, bType), msgAndArgs...) + } + + return Equal(t, expected, actual, msgAndArgs...) +} + +// NotNil asserts that the specified object is not nil. +// +// assert.NotNil(t, err) +func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + if !isNil(object) { + return true + } + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, "Expected value not to be nil.", msgAndArgs...) +} + +// isNil checks if a specified object is nil or not, without Failing. +func isNil(object interface{}) bool { + if object == nil { + return true + } + + value := reflect.ValueOf(object) + switch value.Kind() { + case + reflect.Chan, reflect.Func, + reflect.Interface, reflect.Map, + reflect.Ptr, reflect.Slice, reflect.UnsafePointer: + + return value.IsNil() + } + + return false +} + +// Nil asserts that the specified object is nil. +// +// assert.Nil(t, err) +func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + if isNil(object) { + return true + } + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, fmt.Sprintf("Expected nil, but got: %#v", object), msgAndArgs...) +} + +// isEmpty gets whether the specified object is considered empty or not. +func isEmpty(object interface{}) bool { + // get nil case out of the way + if object == nil { + return true + } + + return isEmptyValue(reflect.ValueOf(object)) +} + +// isEmptyValue gets whether the specified reflect.Value is considered empty or not. +func isEmptyValue(objValue reflect.Value) bool { + if objValue.IsZero() { + return true + } + // Special cases of non-zero values that we consider empty + switch objValue.Kind() { + // collection types are empty when they have no element + // Note: array types are empty when they match their zero-initialized state. + case reflect.Chan, reflect.Map, reflect.Slice: + return objValue.Len() == 0 + // non-nil pointers are empty if the value they point to is empty + case reflect.Ptr: + return isEmptyValue(objValue.Elem()) + } + return false +} + +// Empty asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". +// +// assert.Empty(t, obj) +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value +func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + pass := isEmpty(object) + if !pass { + if h, ok := t.(tHelper); ok { + h.Helper() + } + Fail(t, fmt.Sprintf("Should be empty, but was %v", object), msgAndArgs...) + } + + return pass +} + +// NotEmpty asserts that the specified object is NOT [Empty]. +// +// if assert.NotEmpty(t, obj) { +// assert.Equal(t, "two", obj[1]) +// } +func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + pass := !isEmpty(object) + if !pass { + if h, ok := t.(tHelper); ok { + h.Helper() + } + Fail(t, fmt.Sprintf("Should NOT be empty, but was %v", object), msgAndArgs...) + } + + return pass +} + +// getLen tries to get the length of an object. +// It returns (0, false) if impossible. +func getLen(x interface{}) (length int, ok bool) { + v := reflect.ValueOf(x) + defer func() { + ok = recover() == nil + }() + return v.Len(), true +} + +// Len asserts that the specified object has specific length. +// Len also fails if the object has a type that len() not accept. +// +// assert.Len(t, mySlice, 3) +func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + l, ok := getLen(object) + if !ok { + return Fail(t, fmt.Sprintf("\"%v\" could not be applied builtin len()", object), msgAndArgs...) + } + + if l != length { + return Fail(t, fmt.Sprintf("\"%v\" should have %d item(s), but has %d", object, length, l), msgAndArgs...) + } + return true +} + +// True asserts that the specified value is true. +// +// assert.True(t, myBool) +func True(t TestingT, value bool, msgAndArgs ...interface{}) bool { + if !value { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, "Should be true", msgAndArgs...) + } + + return true +} + +// False asserts that the specified value is false. +// +// assert.False(t, myBool) +func False(t TestingT, value bool, msgAndArgs ...interface{}) bool { + if value { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, "Should be false", msgAndArgs...) + } + + return true +} + +// NotEqual asserts that the specified values are NOT equal. +// +// assert.NotEqual(t, obj1, obj2) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). +func NotEqual(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if err := validateEqualArgs(expected, actual); err != nil { + return Fail(t, fmt.Sprintf("Invalid operation: %#v != %#v (%s)", + expected, actual, err), msgAndArgs...) + } + + if ObjectsAreEqual(expected, actual) { + return Fail(t, fmt.Sprintf("Should not be: %#v\n", actual), msgAndArgs...) + } + + return true +} + +// NotEqualValues asserts that two objects are not equal even when converted to the same type +// +// assert.NotEqualValues(t, obj1, obj2) +func NotEqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + if ObjectsAreEqualValues(expected, actual) { + return Fail(t, fmt.Sprintf("Should not be: %#v\n", actual), msgAndArgs...) + } + + return true +} + +// containsElement try loop over the list check if the list includes the element. +// return (false, false) if impossible. +// return (true, false) if element was not found. +// return (true, true) if element was found. +func containsElement(list interface{}, element interface{}) (ok, found bool) { + listValue := reflect.ValueOf(list) + listType := reflect.TypeOf(list) + if listType == nil { + return false, false + } + listKind := listType.Kind() + defer func() { + if e := recover(); e != nil { + ok = false + found = false + } + }() + + if listKind == reflect.String { + elementValue := reflect.ValueOf(element) + return true, strings.Contains(listValue.String(), elementValue.String()) + } + + if listKind == reflect.Map { + mapKeys := listValue.MapKeys() + for i := 0; i < len(mapKeys); i++ { + if ObjectsAreEqual(mapKeys[i].Interface(), element) { + return true, true + } + } + return true, false + } + + for i := 0; i < listValue.Len(); i++ { + if ObjectsAreEqual(listValue.Index(i).Interface(), element) { + return true, true + } + } + return true, false +} + +// Contains asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// assert.Contains(t, "Hello World", "World") +// assert.Contains(t, ["Hello", "World"], "World") +// assert.Contains(t, {"Hello": "World"}, "Hello") +func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + ok, found := containsElement(s, contains) + if !ok { + return Fail(t, fmt.Sprintf("%#v could not be applied builtin len()", s), msgAndArgs...) + } + if !found { + return Fail(t, fmt.Sprintf("%#v does not contain %#v", s, contains), msgAndArgs...) + } + + return true +} + +// NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// assert.NotContains(t, "Hello World", "Earth") +// assert.NotContains(t, ["Hello", "World"], "Earth") +// assert.NotContains(t, {"Hello": "World"}, "Earth") +func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + ok, found := containsElement(s, contains) + if !ok { + return Fail(t, fmt.Sprintf("%#v could not be applied builtin len()", s), msgAndArgs...) + } + if found { + return Fail(t, fmt.Sprintf("%#v should not contain %#v", s, contains), msgAndArgs...) + } + + return true +} + +// Subset asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// assert.Subset(t, [1, 2, 3], [1, 2]) +// assert.Subset(t, {"x": 1, "y": 2}, {"x": 1}) +// assert.Subset(t, [1, 2, 3], {1: "one", 2: "two"}) +// assert.Subset(t, {"x": 1, "y": 2}, ["x"]) +func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if subset == nil { + return true // we consider nil to be equal to the nil set + } + + listKind := reflect.TypeOf(list).Kind() + if listKind != reflect.Array && listKind != reflect.Slice && listKind != reflect.Map { + return Fail(t, fmt.Sprintf("%q has an unsupported type %s", list, listKind), msgAndArgs...) + } + + subsetKind := reflect.TypeOf(subset).Kind() + if subsetKind != reflect.Array && subsetKind != reflect.Slice && subsetKind != reflect.Map { + return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...) + } + + if subsetKind == reflect.Map && listKind == reflect.Map { + subsetMap := reflect.ValueOf(subset) + actualMap := reflect.ValueOf(list) + + for _, k := range subsetMap.MapKeys() { + ev := subsetMap.MapIndex(k) + av := actualMap.MapIndex(k) + + if !av.IsValid() { + return Fail(t, fmt.Sprintf("%#v does not contain %#v", list, subset), msgAndArgs...) + } + if !ObjectsAreEqual(ev.Interface(), av.Interface()) { + return Fail(t, fmt.Sprintf("%#v does not contain %#v", list, subset), msgAndArgs...) + } + } + + return true + } + + subsetList := reflect.ValueOf(subset) + if subsetKind == reflect.Map { + keys := make([]interface{}, subsetList.Len()) + for idx, key := range subsetList.MapKeys() { + keys[idx] = key.Interface() + } + subsetList = reflect.ValueOf(keys) + } + for i := 0; i < subsetList.Len(); i++ { + element := subsetList.Index(i).Interface() + ok, found := containsElement(list, element) + if !ok { + return Fail(t, fmt.Sprintf("%#v could not be applied builtin len()", list), msgAndArgs...) + } + if !found { + return Fail(t, fmt.Sprintf("%#v does not contain %#v", list, element), msgAndArgs...) + } + } + + return true +} + +// NotSubset asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// assert.NotSubset(t, [1, 3, 4], [1, 2]) +// assert.NotSubset(t, {"x": 1, "y": 2}, {"z": 3}) +// assert.NotSubset(t, [1, 3, 4], {1: "one", 2: "two"}) +// assert.NotSubset(t, {"x": 1, "y": 2}, ["z"]) +func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if subset == nil { + return Fail(t, "nil is the empty set which is a subset of every set", msgAndArgs...) + } + + listKind := reflect.TypeOf(list).Kind() + if listKind != reflect.Array && listKind != reflect.Slice && listKind != reflect.Map { + return Fail(t, fmt.Sprintf("%q has an unsupported type %s", list, listKind), msgAndArgs...) + } + + subsetKind := reflect.TypeOf(subset).Kind() + if subsetKind != reflect.Array && subsetKind != reflect.Slice && subsetKind != reflect.Map { + return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...) + } + + if subsetKind == reflect.Map && listKind == reflect.Map { + subsetMap := reflect.ValueOf(subset) + actualMap := reflect.ValueOf(list) + + for _, k := range subsetMap.MapKeys() { + ev := subsetMap.MapIndex(k) + av := actualMap.MapIndex(k) + + if !av.IsValid() { + return true + } + if !ObjectsAreEqual(ev.Interface(), av.Interface()) { + return true + } + } + + return Fail(t, fmt.Sprintf("%q is a subset of %q", subset, list), msgAndArgs...) + } + + subsetList := reflect.ValueOf(subset) + if subsetKind == reflect.Map { + keys := make([]interface{}, subsetList.Len()) + for idx, key := range subsetList.MapKeys() { + keys[idx] = key.Interface() + } + subsetList = reflect.ValueOf(keys) + } + for i := 0; i < subsetList.Len(); i++ { + element := subsetList.Index(i).Interface() + ok, found := containsElement(list, element) + if !ok { + return Fail(t, fmt.Sprintf("%q could not be applied builtin len()", list), msgAndArgs...) + } + if !found { + return true + } + } + + return Fail(t, fmt.Sprintf("%q is a subset of %q", subset, list), msgAndArgs...) +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// assert.ElementsMatch(t, [1, 3, 2, 3], [1, 3, 3, 2]) +func ElementsMatch(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if isEmpty(listA) && isEmpty(listB) { + return true + } + + if !isList(t, listA, msgAndArgs...) || !isList(t, listB, msgAndArgs...) { + return false + } + + extraA, extraB := diffLists(listA, listB) + + if len(extraA) == 0 && len(extraB) == 0 { + return true + } + + return Fail(t, formatListDiff(listA, listB, extraA, extraB), msgAndArgs...) +} + +// isList checks that the provided value is array or slice. +func isList(t TestingT, list interface{}, msgAndArgs ...interface{}) (ok bool) { + kind := reflect.TypeOf(list).Kind() + if kind != reflect.Array && kind != reflect.Slice { + return Fail(t, fmt.Sprintf("%q has an unsupported type %s, expecting array or slice", list, kind), + msgAndArgs...) + } + return true +} + +// diffLists diffs two arrays/slices and returns slices of elements that are only in A and only in B. +// If some element is present multiple times, each instance is counted separately (e.g. if something is 2x in A and +// 5x in B, it will be 0x in extraA and 3x in extraB). The order of items in both lists is ignored. +func diffLists(listA, listB interface{}) (extraA, extraB []interface{}) { + aValue := reflect.ValueOf(listA) + bValue := reflect.ValueOf(listB) + + aLen := aValue.Len() + bLen := bValue.Len() + + // Mark indexes in bValue that we already used + visited := make([]bool, bLen) + for i := 0; i < aLen; i++ { + element := aValue.Index(i).Interface() + found := false + for j := 0; j < bLen; j++ { + if visited[j] { + continue + } + if ObjectsAreEqual(bValue.Index(j).Interface(), element) { + visited[j] = true + found = true + break + } + } + if !found { + extraA = append(extraA, element) + } + } + + for j := 0; j < bLen; j++ { + if visited[j] { + continue + } + extraB = append(extraB, bValue.Index(j).Interface()) + } + + return +} + +func formatListDiff(listA, listB interface{}, extraA, extraB []interface{}) string { + var msg bytes.Buffer + + msg.WriteString("elements differ") + if len(extraA) > 0 { + msg.WriteString("\n\nextra elements in list A:\n") + msg.WriteString(spewConfig.Sdump(extraA)) + } + if len(extraB) > 0 { + msg.WriteString("\n\nextra elements in list B:\n") + msg.WriteString(spewConfig.Sdump(extraB)) + } + msg.WriteString("\n\nlistA:\n") + msg.WriteString(spewConfig.Sdump(listA)) + msg.WriteString("\n\nlistB:\n") + msg.WriteString(spewConfig.Sdump(listB)) + + return msg.String() +} + +// NotElementsMatch asserts that the specified listA(array, slice...) is NOT equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should not match. +// This is an inverse of ElementsMatch. +// +// assert.NotElementsMatch(t, [1, 1, 2, 3], [1, 1, 2, 3]) -> false +// +// assert.NotElementsMatch(t, [1, 1, 2, 3], [1, 2, 3]) -> true +// +// assert.NotElementsMatch(t, [1, 2, 3], [1, 2, 4]) -> true +func NotElementsMatch(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if isEmpty(listA) && isEmpty(listB) { + return Fail(t, "listA and listB contain the same elements", msgAndArgs) + } + + if !isList(t, listA, msgAndArgs...) { + return Fail(t, "listA is not a list type", msgAndArgs...) + } + if !isList(t, listB, msgAndArgs...) { + return Fail(t, "listB is not a list type", msgAndArgs...) + } + + extraA, extraB := diffLists(listA, listB) + if len(extraA) == 0 && len(extraB) == 0 { + return Fail(t, "listA and listB contain the same elements", msgAndArgs) + } + + return true +} + +// Condition uses a Comparison to assert a complex condition. +func Condition(t TestingT, comp Comparison, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + result := comp() + if !result { + Fail(t, "Condition failed!", msgAndArgs...) + } + return result +} + +// PanicTestFunc defines a func that should be passed to the assert.Panics and assert.NotPanics +// methods, and represents a simple func that takes no arguments, and returns nothing. +type PanicTestFunc func() + +// didPanic returns true if the function passed to it panics. Otherwise, it returns false. +func didPanic(f PanicTestFunc) (didPanic bool, message interface{}, stack string) { + didPanic = true + + defer func() { + message = recover() + if didPanic { + stack = string(debug.Stack()) + } + }() + + // call the target function + f() + didPanic = false + + return +} + +// Panics asserts that the code inside the specified PanicTestFunc panics. +// +// assert.Panics(t, func(){ GoCrazy() }) +func Panics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + if funcDidPanic, panicValue, _ := didPanic(f); !funcDidPanic { + return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%#v", f, panicValue), msgAndArgs...) + } + + return true +} + +// PanicsWithValue asserts that the code inside the specified PanicTestFunc panics, and that +// the recovered panic value equals the expected panic value. +// +// assert.PanicsWithValue(t, "crazy error", func(){ GoCrazy() }) +func PanicsWithValue(t TestingT, expected interface{}, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + funcDidPanic, panicValue, panickedStack := didPanic(f) + if !funcDidPanic { + return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%#v", f, panicValue), msgAndArgs...) + } + if panicValue != expected { + return Fail(t, fmt.Sprintf("func %#v should panic with value:\t%#v\n\tPanic value:\t%#v\n\tPanic stack:\t%s", f, expected, panicValue, panickedStack), msgAndArgs...) + } + + return true +} + +// PanicsWithError asserts that the code inside the specified PanicTestFunc +// panics, and that the recovered panic value is an error that satisfies the +// EqualError comparison. +// +// assert.PanicsWithError(t, "crazy error", func(){ GoCrazy() }) +func PanicsWithError(t TestingT, errString string, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + funcDidPanic, panicValue, panickedStack := didPanic(f) + if !funcDidPanic { + return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%#v", f, panicValue), msgAndArgs...) + } + panicErr, ok := panicValue.(error) + if !ok || panicErr.Error() != errString { + return Fail(t, fmt.Sprintf("func %#v should panic with error message:\t%#v\n\tPanic value:\t%#v\n\tPanic stack:\t%s", f, errString, panicValue, panickedStack), msgAndArgs...) + } + + return true +} + +// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// assert.NotPanics(t, func(){ RemainCalm() }) +func NotPanics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + if funcDidPanic, panicValue, panickedStack := didPanic(f); funcDidPanic { + return Fail(t, fmt.Sprintf("func %#v should not panic\n\tPanic value:\t%v\n\tPanic stack:\t%s", f, panicValue, panickedStack), msgAndArgs...) + } + + return true +} + +// WithinDuration asserts that the two times are within duration delta of each other. +// +// assert.WithinDuration(t, time.Now(), time.Now(), 10*time.Second) +func WithinDuration(t TestingT, expected, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + dt := expected.Sub(actual) + if dt < -delta || dt > delta { + return Fail(t, fmt.Sprintf("Max difference between %v and %v allowed is %v, but difference was %v", expected, actual, delta, dt), msgAndArgs...) + } + + return true +} + +// WithinRange asserts that a time is within a time range (inclusive). +// +// assert.WithinRange(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) +func WithinRange(t TestingT, actual, start, end time.Time, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + if end.Before(start) { + return Fail(t, "Start should be before end", msgAndArgs...) + } + + if actual.Before(start) { + return Fail(t, fmt.Sprintf("Time %v expected to be in time range %v to %v, but is before the range", actual, start, end), msgAndArgs...) + } else if actual.After(end) { + return Fail(t, fmt.Sprintf("Time %v expected to be in time range %v to %v, but is after the range", actual, start, end), msgAndArgs...) + } + + return true +} + +func toFloat(x interface{}) (float64, bool) { + var xf float64 + xok := true + + switch xn := x.(type) { + case uint: + xf = float64(xn) + case uint8: + xf = float64(xn) + case uint16: + xf = float64(xn) + case uint32: + xf = float64(xn) + case uint64: + xf = float64(xn) + case int: + xf = float64(xn) + case int8: + xf = float64(xn) + case int16: + xf = float64(xn) + case int32: + xf = float64(xn) + case int64: + xf = float64(xn) + case float32: + xf = float64(xn) + case float64: + xf = xn + case time.Duration: + xf = float64(xn) + default: + xok = false + } + + return xf, xok +} + +// InDelta asserts that the two numerals are within delta of each other. +// +// assert.InDelta(t, math.Pi, 22/7.0, 0.01) +func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + af, aok := toFloat(expected) + bf, bok := toFloat(actual) + + if !aok || !bok { + return Fail(t, "Parameters must be numerical", msgAndArgs...) + } + + if math.IsNaN(af) && math.IsNaN(bf) { + return true + } + + if math.IsNaN(af) { + return Fail(t, "Expected must not be NaN", msgAndArgs...) + } + + if math.IsNaN(bf) { + return Fail(t, fmt.Sprintf("Expected %v with delta %v, but was NaN", expected, delta), msgAndArgs...) + } + + dt := af - bf + if dt < -delta || dt > delta { + return Fail(t, fmt.Sprintf("Max difference between %v and %v allowed is %v, but difference was %v", expected, actual, delta, dt), msgAndArgs...) + } + + return true +} + +// InDeltaSlice is the same as InDelta, except it compares two slices. +func InDeltaSlice(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if expected == nil || actual == nil || + reflect.TypeOf(actual).Kind() != reflect.Slice || + reflect.TypeOf(expected).Kind() != reflect.Slice { + return Fail(t, "Parameters must be slice", msgAndArgs...) + } + + actualSlice := reflect.ValueOf(actual) + expectedSlice := reflect.ValueOf(expected) + + for i := 0; i < actualSlice.Len(); i++ { + result := InDelta(t, actualSlice.Index(i).Interface(), expectedSlice.Index(i).Interface(), delta, msgAndArgs...) + if !result { + return result + } + } + + return true +} + +// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func InDeltaMapValues(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if expected == nil || actual == nil || + reflect.TypeOf(actual).Kind() != reflect.Map || + reflect.TypeOf(expected).Kind() != reflect.Map { + return Fail(t, "Arguments must be maps", msgAndArgs...) + } + + expectedMap := reflect.ValueOf(expected) + actualMap := reflect.ValueOf(actual) + + if expectedMap.Len() != actualMap.Len() { + return Fail(t, "Arguments must have the same number of keys", msgAndArgs...) + } + + for _, k := range expectedMap.MapKeys() { + ev := expectedMap.MapIndex(k) + av := actualMap.MapIndex(k) + + if !ev.IsValid() { + return Fail(t, fmt.Sprintf("missing key %q in expected map", k), msgAndArgs...) + } + + if !av.IsValid() { + return Fail(t, fmt.Sprintf("missing key %q in actual map", k), msgAndArgs...) + } + + if !InDelta( + t, + ev.Interface(), + av.Interface(), + delta, + msgAndArgs..., + ) { + return false + } + } + + return true +} + +func calcRelativeError(expected, actual interface{}) (float64, error) { + af, aok := toFloat(expected) + bf, bok := toFloat(actual) + if !aok || !bok { + return 0, fmt.Errorf("Parameters must be numerical") + } + if math.IsNaN(af) && math.IsNaN(bf) { + return 0, nil + } + if math.IsNaN(af) { + return 0, errors.New("expected value must not be NaN") + } + if af == 0 { + return 0, fmt.Errorf("expected value must have a value other than zero to calculate the relative error") + } + if math.IsNaN(bf) { + return 0, errors.New("actual value must not be NaN") + } + + return math.Abs(af-bf) / math.Abs(af), nil +} + +// InEpsilon asserts that expected and actual have a relative error less than epsilon +func InEpsilon(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if math.IsNaN(epsilon) { + return Fail(t, "epsilon must not be NaN", msgAndArgs...) + } + actualEpsilon, err := calcRelativeError(expected, actual) + if err != nil { + return Fail(t, err.Error(), msgAndArgs...) + } + if math.IsNaN(actualEpsilon) { + return Fail(t, "relative error is NaN", msgAndArgs...) + } + if actualEpsilon > epsilon { + return Fail(t, fmt.Sprintf("Relative error is too high: %#v (expected)\n"+ + " < %#v (actual)", epsilon, actualEpsilon), msgAndArgs...) + } + + return true +} + +// InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. +func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + if expected == nil || actual == nil { + return Fail(t, "Parameters must be slice", msgAndArgs...) + } + + expectedSlice := reflect.ValueOf(expected) + actualSlice := reflect.ValueOf(actual) + + if expectedSlice.Type().Kind() != reflect.Slice { + return Fail(t, "Expected value must be slice", msgAndArgs...) + } + + expectedLen := expectedSlice.Len() + if !IsType(t, expected, actual) || !Len(t, actual, expectedLen) { + return false + } + + for i := 0; i < expectedLen; i++ { + if !InEpsilon(t, expectedSlice.Index(i).Interface(), actualSlice.Index(i).Interface(), epsilon, "at index %d", i) { + return false + } + } + + return true +} + +/* + Errors +*/ + +// NoError asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if assert.NoError(t, err) { +// assert.Equal(t, expectedObj, actualObj) +// } +func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool { + if err != nil { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, fmt.Sprintf("Received unexpected error:\n%+v", err), msgAndArgs...) + } + + return true +} + +// Error asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// assert.Error(t, err) +func Error(t TestingT, err error, msgAndArgs ...interface{}) bool { + if err == nil { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, "An error is expected but got nil.", msgAndArgs...) + } + + return true +} + +// EqualError asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// assert.EqualError(t, err, expectedErrorString) +func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !Error(t, theError, msgAndArgs...) { + return false + } + expected := errString + actual := theError.Error() + // don't need to use deep equals here, we know they are both strings + if expected != actual { + return Fail(t, fmt.Sprintf("Error message not equal:\n"+ + "expected: %q\n"+ + "actual : %q", expected, actual), msgAndArgs...) + } + return true +} + +// ErrorContains asserts that a function returned an error (i.e. not `nil`) +// and that the error contains the specified substring. +// +// actualObj, err := SomeFunction() +// assert.ErrorContains(t, err, expectedErrorSubString) +func ErrorContains(t TestingT, theError error, contains string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !Error(t, theError, msgAndArgs...) { + return false + } + + actual := theError.Error() + if !strings.Contains(actual, contains) { + return Fail(t, fmt.Sprintf("Error %#v does not contain %#v", actual, contains), msgAndArgs...) + } + + return true +} + +// matchRegexp return true if a specified regexp matches a string. +func matchRegexp(rx interface{}, str interface{}) bool { + var r *regexp.Regexp + if rr, ok := rx.(*regexp.Regexp); ok { + r = rr + } else { + r = regexp.MustCompile(fmt.Sprint(rx)) + } + + switch v := str.(type) { + case []byte: + return r.Match(v) + case string: + return r.MatchString(v) + default: + return r.MatchString(fmt.Sprint(v)) + } +} + +// Regexp asserts that a specified regexp matches a string. +// +// assert.Regexp(t, regexp.MustCompile("start"), "it's starting") +// assert.Regexp(t, "start...$", "it's not starting") +func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + match := matchRegexp(rx, str) + + if !match { + Fail(t, fmt.Sprintf("Expect \"%v\" to match \"%v\"", str, rx), msgAndArgs...) + } + + return match +} + +// NotRegexp asserts that a specified regexp does not match a string. +// +// assert.NotRegexp(t, regexp.MustCompile("starts"), "it's starting") +// assert.NotRegexp(t, "^start", "it's not starting") +func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + match := matchRegexp(rx, str) + + if match { + Fail(t, fmt.Sprintf("Expect \"%v\" to NOT match \"%v\"", str, rx), msgAndArgs...) + } + + return !match +} + +// Zero asserts that i is the zero value for its type. +func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if i != nil && !reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) { + return Fail(t, fmt.Sprintf("Should be zero, but was %v", i), msgAndArgs...) + } + return true +} + +// NotZero asserts that i is not the zero value for its type. +func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if i == nil || reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) { + return Fail(t, fmt.Sprintf("Should not be zero, but was %v", i), msgAndArgs...) + } + return true +} + +// FileExists checks whether a file exists in the given path. It also fails if +// the path points to a directory or there is an error when trying to check the file. +func FileExists(t TestingT, path string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + info, err := os.Lstat(path) + if err != nil { + if os.IsNotExist(err) { + return Fail(t, fmt.Sprintf("unable to find file %q", path), msgAndArgs...) + } + return Fail(t, fmt.Sprintf("error when running os.Lstat(%q): %s", path, err), msgAndArgs...) + } + if info.IsDir() { + return Fail(t, fmt.Sprintf("%q is a directory", path), msgAndArgs...) + } + return true +} + +// NoFileExists checks whether a file does not exist in a given path. It fails +// if the path points to an existing _file_ only. +func NoFileExists(t TestingT, path string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + info, err := os.Lstat(path) + if err != nil { + return true + } + if info.IsDir() { + return true + } + return Fail(t, fmt.Sprintf("file %q exists", path), msgAndArgs...) +} + +// DirExists checks whether a directory exists in the given path. It also fails +// if the path is a file rather a directory or there is an error checking whether it exists. +func DirExists(t TestingT, path string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + info, err := os.Lstat(path) + if err != nil { + if os.IsNotExist(err) { + return Fail(t, fmt.Sprintf("unable to find file %q", path), msgAndArgs...) + } + return Fail(t, fmt.Sprintf("error when running os.Lstat(%q): %s", path, err), msgAndArgs...) + } + if !info.IsDir() { + return Fail(t, fmt.Sprintf("%q is a file", path), msgAndArgs...) + } + return true +} + +// NoDirExists checks whether a directory does not exist in the given path. +// It fails if the path points to an existing _directory_ only. +func NoDirExists(t TestingT, path string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + info, err := os.Lstat(path) + if err != nil { + if os.IsNotExist(err) { + return true + } + return true + } + if !info.IsDir() { + return true + } + return Fail(t, fmt.Sprintf("directory %q exists", path), msgAndArgs...) +} + +// JSONEq asserts that two JSON strings are equivalent. +// +// assert.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) +func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + var expectedJSONAsInterface, actualJSONAsInterface interface{} + + if err := json.Unmarshal([]byte(expected), &expectedJSONAsInterface); err != nil { + return Fail(t, fmt.Sprintf("Expected value ('%s') is not valid json.\nJSON parsing error: '%s'", expected, err.Error()), msgAndArgs...) + } + + // Shortcut if same bytes + if actual == expected { + return true + } + + if err := json.Unmarshal([]byte(actual), &actualJSONAsInterface); err != nil { + return Fail(t, fmt.Sprintf("Input ('%s') needs to be valid json.\nJSON parsing error: '%s'", actual, err.Error()), msgAndArgs...) + } + + return Equal(t, expectedJSONAsInterface, actualJSONAsInterface, msgAndArgs...) +} + +// YAMLEq asserts that two YAML strings are equivalent. +func YAMLEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + var expectedYAMLAsInterface, actualYAMLAsInterface interface{} + + if err := yaml.Unmarshal([]byte(expected), &expectedYAMLAsInterface); err != nil { + return Fail(t, fmt.Sprintf("Expected value ('%s') is not valid yaml.\nYAML parsing error: '%s'", expected, err.Error()), msgAndArgs...) + } + + // Shortcut if same bytes + if actual == expected { + return true + } + + if err := yaml.Unmarshal([]byte(actual), &actualYAMLAsInterface); err != nil { + return Fail(t, fmt.Sprintf("Input ('%s') needs to be valid yaml.\nYAML error: '%s'", actual, err.Error()), msgAndArgs...) + } + + return Equal(t, expectedYAMLAsInterface, actualYAMLAsInterface, msgAndArgs...) +} + +func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { + t := reflect.TypeOf(v) + k := t.Kind() + + if k == reflect.Ptr { + t = t.Elem() + k = t.Kind() + } + return t, k +} + +// diff returns a diff of both values as long as both are of the same type and +// are a struct, map, slice, array or string. Otherwise it returns an empty string. +func diff(expected interface{}, actual interface{}) string { + if expected == nil || actual == nil { + return "" + } + + et, ek := typeAndKind(expected) + at, _ := typeAndKind(actual) + + if et != at { + return "" + } + + if ek != reflect.Struct && ek != reflect.Map && ek != reflect.Slice && ek != reflect.Array && ek != reflect.String { + return "" + } + + var e, a string + + switch et { + case reflect.TypeOf(""): + e = reflect.ValueOf(expected).String() + a = reflect.ValueOf(actual).String() + case reflect.TypeOf(time.Time{}): + e = spewConfigStringerEnabled.Sdump(expected) + a = spewConfigStringerEnabled.Sdump(actual) + default: + e = spewConfig.Sdump(expected) + a = spewConfig.Sdump(actual) + } + + diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(e), + B: difflib.SplitLines(a), + FromFile: "Expected", + FromDate: "", + ToFile: "Actual", + ToDate: "", + Context: 1, + }) + + return "\n\nDiff:\n" + diff +} + +func isFunction(arg interface{}) bool { + if arg == nil { + return false + } + return reflect.TypeOf(arg).Kind() == reflect.Func +} + +var spewConfig = spew.ConfigState{ + Indent: " ", + DisablePointerAddresses: true, + DisableCapacities: true, + SortKeys: true, + DisableMethods: true, + MaxDepth: 10, +} + +var spewConfigStringerEnabled = spew.ConfigState{ + Indent: " ", + DisablePointerAddresses: true, + DisableCapacities: true, + SortKeys: true, + MaxDepth: 10, +} + +type tHelper = interface { + Helper() +} + +// Eventually asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// assert.Eventually(t, func() bool { return true; }, time.Second, 10*time.Millisecond) +func Eventually(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + ch := make(chan bool, 1) + checkCond := func() { ch <- condition() } + + timer := time.NewTimer(waitFor) + defer timer.Stop() + + ticker := time.NewTicker(tick) + defer ticker.Stop() + + var tickC <-chan time.Time + + // Check the condition once first on the initial call. + go checkCond() + + for { + select { + case <-timer.C: + return Fail(t, "Condition never satisfied", msgAndArgs...) + case <-tickC: + tickC = nil + go checkCond() + case v := <-ch: + if v { + return true + } + tickC = ticker.C + } + } +} + +// CollectT implements the TestingT interface and collects all errors. +type CollectT struct { + // A slice of errors. Non-nil slice denotes a failure. + // If it's non-nil but len(c.errors) == 0, this is also a failure + // obtained by direct c.FailNow() call. + errors []error +} + +// Helper is like [testing.T.Helper] but does nothing. +func (CollectT) Helper() {} + +// Errorf collects the error. +func (c *CollectT) Errorf(format string, args ...interface{}) { + c.errors = append(c.errors, fmt.Errorf(format, args...)) +} + +// FailNow stops execution by calling runtime.Goexit. +func (c *CollectT) FailNow() { + c.fail() + runtime.Goexit() +} + +// Deprecated: That was a method for internal usage that should not have been published. Now just panics. +func (*CollectT) Reset() { + panic("Reset() is deprecated") +} + +// Deprecated: That was a method for internal usage that should not have been published. Now just panics. +func (*CollectT) Copy(TestingT) { + panic("Copy() is deprecated") +} + +func (c *CollectT) fail() { + if !c.failed() { + c.errors = []error{} // Make it non-nil to mark a failure. + } +} + +func (c *CollectT) failed() bool { + return c.errors != nil +} + +// EventuallyWithT asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. In contrast to Eventually, +// it supplies a CollectT to the condition function, so that the condition +// function can use the CollectT to call other assertions. +// The condition is considered "met" if no errors are raised in a tick. +// The supplied CollectT collects all errors from one tick (if there are any). +// If the condition is not met before waitFor, the collected errors of +// the last tick are copied to t. +// +// externalValue := false +// go func() { +// time.Sleep(8*time.Second) +// externalValue = true +// }() +// assert.EventuallyWithT(t, func(c *assert.CollectT) { +// // add assertions as needed; any assertion failure will fail the current tick +// assert.True(c, externalValue, "expected 'externalValue' to be true") +// }, 10*time.Second, 1*time.Second, "external state has not changed to 'true'; still false") +func EventuallyWithT(t TestingT, condition func(collect *CollectT), waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + var lastFinishedTickErrs []error + ch := make(chan *CollectT, 1) + + checkCond := func() { + collect := new(CollectT) + defer func() { + ch <- collect + }() + condition(collect) + } + + timer := time.NewTimer(waitFor) + defer timer.Stop() + + ticker := time.NewTicker(tick) + defer ticker.Stop() + + var tickC <-chan time.Time + + // Check the condition once first on the initial call. + go checkCond() + + for { + select { + case <-timer.C: + for _, err := range lastFinishedTickErrs { + t.Errorf("%v", err) + } + return Fail(t, "Condition never satisfied", msgAndArgs...) + case <-tickC: + tickC = nil + go checkCond() + case collect := <-ch: + if !collect.failed() { + return true + } + // Keep the errors from the last ended condition, so that they can be copied to t if timeout is reached. + lastFinishedTickErrs = collect.errors + tickC = ticker.C + } + } +} + +// Never asserts that the given condition doesn't satisfy in waitFor time, +// periodically checking the target function each tick. +// +// assert.Never(t, func() bool { return false; }, time.Second, 10*time.Millisecond) +func Never(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + ch := make(chan bool, 1) + checkCond := func() { ch <- condition() } + + timer := time.NewTimer(waitFor) + defer timer.Stop() + + ticker := time.NewTicker(tick) + defer ticker.Stop() + + var tickC <-chan time.Time + + // Check the condition once first on the initial call. + go checkCond() + + for { + select { + case <-timer.C: + return true + case <-tickC: + tickC = nil + go checkCond() + case v := <-ch: + if v { + return Fail(t, "Condition satisfied", msgAndArgs...) + } + tickC = ticker.C + } + } +} + +// ErrorIs asserts that at least one of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func ErrorIs(t TestingT, err, target error, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if errors.Is(err, target) { + return true + } + + var expectedText string + if target != nil { + expectedText = target.Error() + if err == nil { + return Fail(t, fmt.Sprintf("Expected error with %q in chain but got nil.", expectedText), msgAndArgs...) + } + } + + chain := buildErrorChainString(err, false) + + return Fail(t, fmt.Sprintf("Target error should be in err chain:\n"+ + "expected: %q\n"+ + "in chain: %s", expectedText, chain, + ), msgAndArgs...) +} + +// NotErrorIs asserts that none of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func NotErrorIs(t TestingT, err, target error, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !errors.Is(err, target) { + return true + } + + var expectedText string + if target != nil { + expectedText = target.Error() + } + + chain := buildErrorChainString(err, false) + + return Fail(t, fmt.Sprintf("Target error should not be in err chain:\n"+ + "found: %q\n"+ + "in chain: %s", expectedText, chain, + ), msgAndArgs...) +} + +// ErrorAs asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. +// This is a wrapper for errors.As. +func ErrorAs(t TestingT, err error, target interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if errors.As(err, target) { + return true + } + + expectedType := reflect.TypeOf(target).Elem().String() + if err == nil { + return Fail(t, fmt.Sprintf("An error is expected but got nil.\n"+ + "expected: %s", expectedType), msgAndArgs...) + } + + chain := buildErrorChainString(err, true) + + return Fail(t, fmt.Sprintf("Should be in error chain:\n"+ + "expected: %s\n"+ + "in chain: %s", expectedType, chain, + ), msgAndArgs...) +} + +// NotErrorAs asserts that none of the errors in err's chain matches target, +// but if so, sets target to that error value. +func NotErrorAs(t TestingT, err error, target interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !errors.As(err, target) { + return true + } + + chain := buildErrorChainString(err, true) + + return Fail(t, fmt.Sprintf("Target error should not be in err chain:\n"+ + "found: %s\n"+ + "in chain: %s", reflect.TypeOf(target).Elem().String(), chain, + ), msgAndArgs...) +} + +func unwrapAll(err error) (errs []error) { + errs = append(errs, err) + switch x := err.(type) { + case interface{ Unwrap() error }: + err = x.Unwrap() + if err == nil { + return + } + errs = append(errs, unwrapAll(err)...) + case interface{ Unwrap() []error }: + for _, err := range x.Unwrap() { + errs = append(errs, unwrapAll(err)...) + } + } + return +} + +func buildErrorChainString(err error, withType bool) string { + if err == nil { + return "" + } + + var chain string + errs := unwrapAll(err) + for i := range errs { + if i != 0 { + chain += "\n\t" + } + chain += fmt.Sprintf("%q", errs[i].Error()) + if withType { + chain += fmt.Sprintf(" (%T)", errs[i]) + } + } + return chain +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/doc.go b/scraper-go/vendor/github.com/stretchr/testify/assert/doc.go new file mode 100644 index 00000000..a0b953aa --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/doc.go @@ -0,0 +1,50 @@ +// Package assert provides a set of comprehensive testing tools for use with the normal Go testing system. +// +// # Note +// +// All functions in this package return a bool value indicating whether the assertion has passed. +// +// # Example Usage +// +// The following is a complete example using assert in a standard test function: +// +// import ( +// "testing" +// "github.com/stretchr/testify/assert" +// ) +// +// func TestSomething(t *testing.T) { +// +// var a string = "Hello" +// var b string = "Hello" +// +// assert.Equal(t, a, b, "The two words should be the same.") +// +// } +// +// if you assert many times, use the format below: +// +// import ( +// "testing" +// "github.com/stretchr/testify/assert" +// ) +// +// func TestSomething(t *testing.T) { +// assert := assert.New(t) +// +// var a string = "Hello" +// var b string = "Hello" +// +// assert.Equal(a, b, "The two words should be the same.") +// } +// +// # Assertions +// +// Assertions allow you to easily write test code, and are global funcs in the `assert` package. +// All assertion functions take, as the first argument, the `*testing.T` object provided by the +// testing framework. This allows the assertion funcs to write the failings and other details to +// the correct place. +// +// Every assertion function also takes an optional string message as the final argument, +// allowing custom error messages to be appended to the message the assertion method outputs. +package assert diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/errors.go b/scraper-go/vendor/github.com/stretchr/testify/assert/errors.go new file mode 100644 index 00000000..ac9dc9d1 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/errors.go @@ -0,0 +1,10 @@ +package assert + +import ( + "errors" +) + +// AnError is an error instance useful for testing. If the code does not care +// about error specifics, and only needs to return the error for example, this +// error should be used to make the test code more readable. +var AnError = errors.New("assert.AnError general error for testing") diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/forward_assertions.go b/scraper-go/vendor/github.com/stretchr/testify/assert/forward_assertions.go new file mode 100644 index 00000000..df189d23 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/forward_assertions.go @@ -0,0 +1,16 @@ +package assert + +// Assertions provides assertion methods around the +// TestingT interface. +type Assertions struct { + t TestingT +} + +// New makes a new Assertions object for the specified TestingT. +func New(t TestingT) *Assertions { + return &Assertions{ + t: t, + } +} + +//go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=assert -template=assertion_forward.go.tmpl -include-format-funcs" diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/http_assertions.go b/scraper-go/vendor/github.com/stretchr/testify/assert/http_assertions.go new file mode 100644 index 00000000..5a6bb75f --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/http_assertions.go @@ -0,0 +1,165 @@ +package assert + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" +) + +// httpCode is a helper that returns HTTP code of the response. It returns -1 and +// an error if building a new request fails. +func httpCode(handler http.HandlerFunc, method, url string, values url.Values) (int, error) { + w := httptest.NewRecorder() + req, err := http.NewRequest(method, url, http.NoBody) + if err != nil { + return -1, err + } + req.URL.RawQuery = values.Encode() + handler(w, req) + return w.Code, nil +} + +// HTTPSuccess asserts that a specified handler returns a success status code. +// +// assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + code, err := httpCode(handler, method, url, values) + if err != nil { + Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err), msgAndArgs...) + } + + isSuccessCode := code >= http.StatusOK && code <= http.StatusPartialContent + if !isSuccessCode { + Fail(t, fmt.Sprintf("Expected HTTP success status code for %q but received %d", url+"?"+values.Encode(), code), msgAndArgs...) + } + + return isSuccessCode +} + +// HTTPRedirect asserts that a specified handler returns a redirect status code. +// +// assert.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + code, err := httpCode(handler, method, url, values) + if err != nil { + Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err), msgAndArgs...) + } + + isRedirectCode := code >= http.StatusMultipleChoices && code <= http.StatusTemporaryRedirect + if !isRedirectCode { + Fail(t, fmt.Sprintf("Expected HTTP redirect status code for %q but received %d", url+"?"+values.Encode(), code), msgAndArgs...) + } + + return isRedirectCode +} + +// HTTPError asserts that a specified handler returns an error status code. +// +// assert.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + code, err := httpCode(handler, method, url, values) + if err != nil { + Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err), msgAndArgs...) + } + + isErrorCode := code >= http.StatusBadRequest + if !isErrorCode { + Fail(t, fmt.Sprintf("Expected HTTP error status code for %q but received %d", url+"?"+values.Encode(), code), msgAndArgs...) + } + + return isErrorCode +} + +// HTTPStatusCode asserts that a specified handler returns a specified status code. +// +// assert.HTTPStatusCode(t, myHandler, "GET", "/notImplemented", nil, 501) +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPStatusCode(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + code, err := httpCode(handler, method, url, values) + if err != nil { + Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err), msgAndArgs...) + } + + successful := code == statuscode + if !successful { + Fail(t, fmt.Sprintf("Expected HTTP status code %d for %q but received %d", statuscode, url+"?"+values.Encode(), code), msgAndArgs...) + } + + return successful +} + +// HTTPBody is a helper that returns HTTP body of the response. It returns +// empty string if building a new request fails. +func HTTPBody(handler http.HandlerFunc, method, url string, values url.Values) string { + w := httptest.NewRecorder() + if len(values) > 0 { + url += "?" + values.Encode() + } + req, err := http.NewRequest(method, url, http.NoBody) + if err != nil { + return "" + } + handler(w, req) + return w.Body.String() +} + +// HTTPBodyContains asserts that a specified handler returns a +// body that contains a string. +// +// assert.HTTPBodyContains(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + body := HTTPBody(handler, method, url, values) + + contains := strings.Contains(body, fmt.Sprint(str)) + if !contains { + Fail(t, fmt.Sprintf("Expected response body for %q to contain %q but found %q", url+"?"+values.Encode(), str, body), msgAndArgs...) + } + + return contains +} + +// HTTPBodyNotContains asserts that a specified handler returns a +// body that does not contain a string. +// +// assert.HTTPBodyNotContains(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + body := HTTPBody(handler, method, url, values) + + contains := strings.Contains(body, fmt.Sprint(str)) + if contains { + Fail(t, fmt.Sprintf("Expected response body for %q to NOT contain %q but found %q", url+"?"+values.Encode(), str, body), msgAndArgs...) + } + + return !contains +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_custom.go b/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_custom.go new file mode 100644 index 00000000..5a74c4f4 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_custom.go @@ -0,0 +1,24 @@ +//go:build testify_yaml_custom && !testify_yaml_fail && !testify_yaml_default + +// Package yaml is an implementation of YAML functions that calls a pluggable implementation. +// +// This implementation is selected with the testify_yaml_custom build tag. +// +// go test -tags testify_yaml_custom +// +// This implementation can be used at build time to replace the default implementation +// to avoid linking with [gopkg.in/yaml.v3]. +// +// In your test package: +// +// import assertYaml "github.com/stretchr/testify/assert/yaml" +// +// func init() { +// assertYaml.Unmarshal = func (in []byte, out interface{}) error { +// // ... +// return nil +// } +// } +package yaml + +var Unmarshal func(in []byte, out interface{}) error diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_default.go b/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_default.go new file mode 100644 index 00000000..0bae80e3 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_default.go @@ -0,0 +1,36 @@ +//go:build !testify_yaml_fail && !testify_yaml_custom + +// Package yaml is just an indirection to handle YAML deserialization. +// +// This package is just an indirection that allows the builder to override the +// indirection with an alternative implementation of this package that uses +// another implementation of YAML deserialization. This allows to not either not +// use YAML deserialization at all, or to use another implementation than +// [gopkg.in/yaml.v3] (for example for license compatibility reasons, see [PR #1120]). +// +// Alternative implementations are selected using build tags: +// +// - testify_yaml_fail: [Unmarshal] always fails with an error +// - testify_yaml_custom: [Unmarshal] is a variable. Caller must initialize it +// before calling any of [github.com/stretchr/testify/assert.YAMLEq] or +// [github.com/stretchr/testify/assert.YAMLEqf]. +// +// Usage: +// +// go test -tags testify_yaml_fail +// +// You can check with "go list" which implementation is linked: +// +// go list -f '{{.Imports}}' github.com/stretchr/testify/assert/yaml +// go list -tags testify_yaml_fail -f '{{.Imports}}' github.com/stretchr/testify/assert/yaml +// go list -tags testify_yaml_custom -f '{{.Imports}}' github.com/stretchr/testify/assert/yaml +// +// [PR #1120]: https://github.com/stretchr/testify/pull/1120 +package yaml + +import goyaml "gopkg.in/yaml.v3" + +// Unmarshal is just a wrapper of [gopkg.in/yaml.v3.Unmarshal]. +func Unmarshal(in []byte, out interface{}) error { + return goyaml.Unmarshal(in, out) +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_fail.go b/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_fail.go new file mode 100644 index 00000000..8041803f --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/assert/yaml/yaml_fail.go @@ -0,0 +1,17 @@ +//go:build testify_yaml_fail && !testify_yaml_custom && !testify_yaml_default + +// Package yaml is an implementation of YAML functions that always fail. +// +// This implementation can be used at build time to replace the default implementation +// to avoid linking with [gopkg.in/yaml.v3]: +// +// go test -tags testify_yaml_fail +package yaml + +import "errors" + +var errNotImplemented = errors.New("YAML functions are not available (see https://pkg.go.dev/github.com/stretchr/testify/assert/yaml)") + +func Unmarshal([]byte, interface{}) error { + return errNotImplemented +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/require/doc.go b/scraper-go/vendor/github.com/stretchr/testify/require/doc.go new file mode 100644 index 00000000..c8e3f94a --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/require/doc.go @@ -0,0 +1,31 @@ +// Package require implements the same assertions as the `assert` package but +// stops test execution when a test fails. +// +// # Example Usage +// +// The following is a complete example using require in a standard test function: +// +// import ( +// "testing" +// "github.com/stretchr/testify/require" +// ) +// +// func TestSomething(t *testing.T) { +// +// var a string = "Hello" +// var b string = "Hello" +// +// require.Equal(t, a, b, "The two words should be the same.") +// +// } +// +// # Assertions +// +// The `require` package have same global functions as in the `assert` package, +// but instead of returning a boolean result they call `t.FailNow()`. +// A consequence of this is that it must be called from the goroutine running +// the test function, not from other goroutines created during the test. +// +// Every assertion function also takes an optional string message as the final argument, +// allowing custom error messages to be appended to the message the assertion method outputs. +package require diff --git a/scraper-go/vendor/github.com/stretchr/testify/require/forward_requirements.go b/scraper-go/vendor/github.com/stretchr/testify/require/forward_requirements.go new file mode 100644 index 00000000..1dcb2338 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/require/forward_requirements.go @@ -0,0 +1,16 @@ +package require + +// Assertions provides assertion methods around the +// TestingT interface. +type Assertions struct { + t TestingT +} + +// New makes a new Assertions object for the specified TestingT. +func New(t TestingT) *Assertions { + return &Assertions{ + t: t, + } +} + +//go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=require -template=require_forward.go.tmpl -include-format-funcs" diff --git a/scraper-go/vendor/github.com/stretchr/testify/require/require.go b/scraper-go/vendor/github.com/stretchr/testify/require/require.go new file mode 100644 index 00000000..2d02f9bc --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/require/require.go @@ -0,0 +1,2180 @@ +// Code generated with github.com/stretchr/testify/_codegen; DO NOT EDIT. + +package require + +import ( + assert "github.com/stretchr/testify/assert" + http "net/http" + url "net/url" + time "time" +) + +// Condition uses a Comparison to assert a complex condition. +func Condition(t TestingT, comp assert.Comparison, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Condition(t, comp, msgAndArgs...) { + return + } + t.FailNow() +} + +// Conditionf uses a Comparison to assert a complex condition. +func Conditionf(t TestingT, comp assert.Comparison, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Conditionf(t, comp, msg, args...) { + return + } + t.FailNow() +} + +// Contains asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// require.Contains(t, "Hello World", "World") +// require.Contains(t, ["Hello", "World"], "World") +// require.Contains(t, {"Hello": "World"}, "Hello") +func Contains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Contains(t, s, contains, msgAndArgs...) { + return + } + t.FailNow() +} + +// Containsf asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// require.Containsf(t, "Hello World", "World", "error message %s", "formatted") +// require.Containsf(t, ["Hello", "World"], "World", "error message %s", "formatted") +// require.Containsf(t, {"Hello": "World"}, "Hello", "error message %s", "formatted") +func Containsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Containsf(t, s, contains, msg, args...) { + return + } + t.FailNow() +} + +// DirExists checks whether a directory exists in the given path. It also fails +// if the path is a file rather a directory or there is an error checking whether it exists. +func DirExists(t TestingT, path string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.DirExists(t, path, msgAndArgs...) { + return + } + t.FailNow() +} + +// DirExistsf checks whether a directory exists in the given path. It also fails +// if the path is a file rather a directory or there is an error checking whether it exists. +func DirExistsf(t TestingT, path string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.DirExistsf(t, path, msg, args...) { + return + } + t.FailNow() +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// require.ElementsMatch(t, [1, 3, 2, 3], [1, 3, 3, 2]) +func ElementsMatch(t TestingT, listA interface{}, listB interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.ElementsMatch(t, listA, listB, msgAndArgs...) { + return + } + t.FailNow() +} + +// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// require.ElementsMatchf(t, [1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") +func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.ElementsMatchf(t, listA, listB, msg, args...) { + return + } + t.FailNow() +} + +// Empty asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". +// +// require.Empty(t, obj) +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value +func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Empty(t, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// Emptyf asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". +// +// require.Emptyf(t, obj, "error message %s", "formatted") +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value +func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Emptyf(t, object, msg, args...) { + return + } + t.FailNow() +} + +// Equal asserts that two objects are equal. +// +// require.Equal(t, 123, 123) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func Equal(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Equal(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// EqualError asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// require.EqualError(t, err, expectedErrorString) +func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.EqualError(t, theError, errString, msgAndArgs...) { + return + } + t.FailNow() +} + +// EqualErrorf asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// require.EqualErrorf(t, err, expectedErrorString, "error message %s", "formatted") +func EqualErrorf(t TestingT, theError error, errString string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.EqualErrorf(t, theError, errString, msg, args...) { + return + } + t.FailNow() +} + +// EqualExportedValues asserts that the types of two objects are equal and their public +// fields are also equal. This is useful for comparing structs that have private fields +// that could potentially differ. +// +// type S struct { +// Exported int +// notExported int +// } +// require.EqualExportedValues(t, S{1, 2}, S{1, 3}) => true +// require.EqualExportedValues(t, S{1, 2}, S{2, 3}) => false +func EqualExportedValues(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.EqualExportedValues(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// EqualExportedValuesf asserts that the types of two objects are equal and their public +// fields are also equal. This is useful for comparing structs that have private fields +// that could potentially differ. +// +// type S struct { +// Exported int +// notExported int +// } +// require.EqualExportedValuesf(t, S{1, 2}, S{1, 3}, "error message %s", "formatted") => true +// require.EqualExportedValuesf(t, S{1, 2}, S{2, 3}, "error message %s", "formatted") => false +func EqualExportedValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.EqualExportedValuesf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// EqualValues asserts that two objects are equal or convertible to the larger +// type and equal. +// +// require.EqualValues(t, uint32(123), int32(123)) +func EqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.EqualValues(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// EqualValuesf asserts that two objects are equal or convertible to the larger +// type and equal. +// +// require.EqualValuesf(t, uint32(123), int32(123), "error message %s", "formatted") +func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.EqualValuesf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// Equalf asserts that two objects are equal. +// +// require.Equalf(t, 123, 123, "error message %s", "formatted") +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Equalf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// Error asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// require.Error(t, err) +func Error(t TestingT, err error, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Error(t, err, msgAndArgs...) { + return + } + t.FailNow() +} + +// ErrorAs asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. +// This is a wrapper for errors.As. +func ErrorAs(t TestingT, err error, target interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.ErrorAs(t, err, target, msgAndArgs...) { + return + } + t.FailNow() +} + +// ErrorAsf asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. +// This is a wrapper for errors.As. +func ErrorAsf(t TestingT, err error, target interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.ErrorAsf(t, err, target, msg, args...) { + return + } + t.FailNow() +} + +// ErrorContains asserts that a function returned an error (i.e. not `nil`) +// and that the error contains the specified substring. +// +// actualObj, err := SomeFunction() +// require.ErrorContains(t, err, expectedErrorSubString) +func ErrorContains(t TestingT, theError error, contains string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.ErrorContains(t, theError, contains, msgAndArgs...) { + return + } + t.FailNow() +} + +// ErrorContainsf asserts that a function returned an error (i.e. not `nil`) +// and that the error contains the specified substring. +// +// actualObj, err := SomeFunction() +// require.ErrorContainsf(t, err, expectedErrorSubString, "error message %s", "formatted") +func ErrorContainsf(t TestingT, theError error, contains string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.ErrorContainsf(t, theError, contains, msg, args...) { + return + } + t.FailNow() +} + +// ErrorIs asserts that at least one of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func ErrorIs(t TestingT, err error, target error, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.ErrorIs(t, err, target, msgAndArgs...) { + return + } + t.FailNow() +} + +// ErrorIsf asserts that at least one of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func ErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.ErrorIsf(t, err, target, msg, args...) { + return + } + t.FailNow() +} + +// Errorf asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// require.Errorf(t, err, "error message %s", "formatted") +func Errorf(t TestingT, err error, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Errorf(t, err, msg, args...) { + return + } + t.FailNow() +} + +// Eventually asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// require.Eventually(t, func() bool { return true; }, time.Second, 10*time.Millisecond) +func Eventually(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Eventually(t, condition, waitFor, tick, msgAndArgs...) { + return + } + t.FailNow() +} + +// EventuallyWithT asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. In contrast to Eventually, +// it supplies a CollectT to the condition function, so that the condition +// function can use the CollectT to call other assertions. +// The condition is considered "met" if no errors are raised in a tick. +// The supplied CollectT collects all errors from one tick (if there are any). +// If the condition is not met before waitFor, the collected errors of +// the last tick are copied to t. +// +// externalValue := false +// go func() { +// time.Sleep(8*time.Second) +// externalValue = true +// }() +// require.EventuallyWithT(t, func(c *require.CollectT) { +// // add assertions as needed; any assertion failure will fail the current tick +// require.True(c, externalValue, "expected 'externalValue' to be true") +// }, 10*time.Second, 1*time.Second, "external state has not changed to 'true'; still false") +func EventuallyWithT(t TestingT, condition func(collect *assert.CollectT), waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.EventuallyWithT(t, condition, waitFor, tick, msgAndArgs...) { + return + } + t.FailNow() +} + +// EventuallyWithTf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. In contrast to Eventually, +// it supplies a CollectT to the condition function, so that the condition +// function can use the CollectT to call other assertions. +// The condition is considered "met" if no errors are raised in a tick. +// The supplied CollectT collects all errors from one tick (if there are any). +// If the condition is not met before waitFor, the collected errors of +// the last tick are copied to t. +// +// externalValue := false +// go func() { +// time.Sleep(8*time.Second) +// externalValue = true +// }() +// require.EventuallyWithTf(t, func(c *require.CollectT, "error message %s", "formatted") { +// // add assertions as needed; any assertion failure will fail the current tick +// require.True(c, externalValue, "expected 'externalValue' to be true") +// }, 10*time.Second, 1*time.Second, "external state has not changed to 'true'; still false") +func EventuallyWithTf(t TestingT, condition func(collect *assert.CollectT), waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.EventuallyWithTf(t, condition, waitFor, tick, msg, args...) { + return + } + t.FailNow() +} + +// Eventuallyf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// require.Eventuallyf(t, func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func Eventuallyf(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Eventuallyf(t, condition, waitFor, tick, msg, args...) { + return + } + t.FailNow() +} + +// Exactly asserts that two objects are equal in value and type. +// +// require.Exactly(t, int32(123), int64(123)) +func Exactly(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Exactly(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// Exactlyf asserts that two objects are equal in value and type. +// +// require.Exactlyf(t, int32(123), int64(123), "error message %s", "formatted") +func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Exactlyf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// Fail reports a failure through +func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Fail(t, failureMessage, msgAndArgs...) { + return + } + t.FailNow() +} + +// FailNow fails test +func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.FailNow(t, failureMessage, msgAndArgs...) { + return + } + t.FailNow() +} + +// FailNowf fails test +func FailNowf(t TestingT, failureMessage string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.FailNowf(t, failureMessage, msg, args...) { + return + } + t.FailNow() +} + +// Failf reports a failure through +func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Failf(t, failureMessage, msg, args...) { + return + } + t.FailNow() +} + +// False asserts that the specified value is false. +// +// require.False(t, myBool) +func False(t TestingT, value bool, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.False(t, value, msgAndArgs...) { + return + } + t.FailNow() +} + +// Falsef asserts that the specified value is false. +// +// require.Falsef(t, myBool, "error message %s", "formatted") +func Falsef(t TestingT, value bool, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Falsef(t, value, msg, args...) { + return + } + t.FailNow() +} + +// FileExists checks whether a file exists in the given path. It also fails if +// the path points to a directory or there is an error when trying to check the file. +func FileExists(t TestingT, path string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.FileExists(t, path, msgAndArgs...) { + return + } + t.FailNow() +} + +// FileExistsf checks whether a file exists in the given path. It also fails if +// the path points to a directory or there is an error when trying to check the file. +func FileExistsf(t TestingT, path string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.FileExistsf(t, path, msg, args...) { + return + } + t.FailNow() +} + +// Greater asserts that the first element is greater than the second +// +// require.Greater(t, 2, 1) +// require.Greater(t, float64(2), float64(1)) +// require.Greater(t, "b", "a") +func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Greater(t, e1, e2, msgAndArgs...) { + return + } + t.FailNow() +} + +// GreaterOrEqual asserts that the first element is greater than or equal to the second +// +// require.GreaterOrEqual(t, 2, 1) +// require.GreaterOrEqual(t, 2, 2) +// require.GreaterOrEqual(t, "b", "a") +// require.GreaterOrEqual(t, "b", "b") +func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.GreaterOrEqual(t, e1, e2, msgAndArgs...) { + return + } + t.FailNow() +} + +// GreaterOrEqualf asserts that the first element is greater than or equal to the second +// +// require.GreaterOrEqualf(t, 2, 1, "error message %s", "formatted") +// require.GreaterOrEqualf(t, 2, 2, "error message %s", "formatted") +// require.GreaterOrEqualf(t, "b", "a", "error message %s", "formatted") +// require.GreaterOrEqualf(t, "b", "b", "error message %s", "formatted") +func GreaterOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.GreaterOrEqualf(t, e1, e2, msg, args...) { + return + } + t.FailNow() +} + +// Greaterf asserts that the first element is greater than the second +// +// require.Greaterf(t, 2, 1, "error message %s", "formatted") +// require.Greaterf(t, float64(2), float64(1), "error message %s", "formatted") +// require.Greaterf(t, "b", "a", "error message %s", "formatted") +func Greaterf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Greaterf(t, e1, e2, msg, args...) { + return + } + t.FailNow() +} + +// HTTPBodyContains asserts that a specified handler returns a +// body that contains a string. +// +// require.HTTPBodyContains(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPBodyContains(t, handler, method, url, values, str, msgAndArgs...) { + return + } + t.FailNow() +} + +// HTTPBodyContainsf asserts that a specified handler returns a +// body that contains a string. +// +// require.HTTPBodyContainsf(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPBodyContainsf(t, handler, method, url, values, str, msg, args...) { + return + } + t.FailNow() +} + +// HTTPBodyNotContains asserts that a specified handler returns a +// body that does not contain a string. +// +// require.HTTPBodyNotContains(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPBodyNotContains(t, handler, method, url, values, str, msgAndArgs...) { + return + } + t.FailNow() +} + +// HTTPBodyNotContainsf asserts that a specified handler returns a +// body that does not contain a string. +// +// require.HTTPBodyNotContainsf(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPBodyNotContainsf(t, handler, method, url, values, str, msg, args...) { + return + } + t.FailNow() +} + +// HTTPError asserts that a specified handler returns an error status code. +// +// require.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPError(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPError(t, handler, method, url, values, msgAndArgs...) { + return + } + t.FailNow() +} + +// HTTPErrorf asserts that a specified handler returns an error status code. +// +// require.HTTPErrorf(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPErrorf(t, handler, method, url, values, msg, args...) { + return + } + t.FailNow() +} + +// HTTPRedirect asserts that a specified handler returns a redirect status code. +// +// require.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPRedirect(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPRedirect(t, handler, method, url, values, msgAndArgs...) { + return + } + t.FailNow() +} + +// HTTPRedirectf asserts that a specified handler returns a redirect status code. +// +// require.HTTPRedirectf(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPRedirectf(t, handler, method, url, values, msg, args...) { + return + } + t.FailNow() +} + +// HTTPStatusCode asserts that a specified handler returns a specified status code. +// +// require.HTTPStatusCode(t, myHandler, "GET", "/notImplemented", nil, 501) +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPStatusCode(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPStatusCode(t, handler, method, url, values, statuscode, msgAndArgs...) { + return + } + t.FailNow() +} + +// HTTPStatusCodef asserts that a specified handler returns a specified status code. +// +// require.HTTPStatusCodef(t, myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPStatusCodef(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPStatusCodef(t, handler, method, url, values, statuscode, msg, args...) { + return + } + t.FailNow() +} + +// HTTPSuccess asserts that a specified handler returns a success status code. +// +// require.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPSuccess(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPSuccess(t, handler, method, url, values, msgAndArgs...) { + return + } + t.FailNow() +} + +// HTTPSuccessf asserts that a specified handler returns a success status code. +// +// require.HTTPSuccessf(t, myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPSuccessf(t, handler, method, url, values, msg, args...) { + return + } + t.FailNow() +} + +// Implements asserts that an object is implemented by the specified interface. +// +// require.Implements(t, (*MyInterface)(nil), new(MyObject)) +func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Implements(t, interfaceObject, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// Implementsf asserts that an object is implemented by the specified interface. +// +// require.Implementsf(t, (*MyInterface)(nil), new(MyObject), "error message %s", "formatted") +func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Implementsf(t, interfaceObject, object, msg, args...) { + return + } + t.FailNow() +} + +// InDelta asserts that the two numerals are within delta of each other. +// +// require.InDelta(t, math.Pi, 22/7.0, 0.01) +func InDelta(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InDelta(t, expected, actual, delta, msgAndArgs...) { + return + } + t.FailNow() +} + +// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func InDeltaMapValues(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InDeltaMapValues(t, expected, actual, delta, msgAndArgs...) { + return + } + t.FailNow() +} + +// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func InDeltaMapValuesf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InDeltaMapValuesf(t, expected, actual, delta, msg, args...) { + return + } + t.FailNow() +} + +// InDeltaSlice is the same as InDelta, except it compares two slices. +func InDeltaSlice(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InDeltaSlice(t, expected, actual, delta, msgAndArgs...) { + return + } + t.FailNow() +} + +// InDeltaSlicef is the same as InDelta, except it compares two slices. +func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InDeltaSlicef(t, expected, actual, delta, msg, args...) { + return + } + t.FailNow() +} + +// InDeltaf asserts that the two numerals are within delta of each other. +// +// require.InDeltaf(t, math.Pi, 22/7.0, 0.01, "error message %s", "formatted") +func InDeltaf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InDeltaf(t, expected, actual, delta, msg, args...) { + return + } + t.FailNow() +} + +// InEpsilon asserts that expected and actual have a relative error less than epsilon +func InEpsilon(t TestingT, expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InEpsilon(t, expected, actual, epsilon, msgAndArgs...) { + return + } + t.FailNow() +} + +// InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. +func InEpsilonSlice(t TestingT, expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InEpsilonSlice(t, expected, actual, epsilon, msgAndArgs...) { + return + } + t.FailNow() +} + +// InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. +func InEpsilonSlicef(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InEpsilonSlicef(t, expected, actual, epsilon, msg, args...) { + return + } + t.FailNow() +} + +// InEpsilonf asserts that expected and actual have a relative error less than epsilon +func InEpsilonf(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.InEpsilonf(t, expected, actual, epsilon, msg, args...) { + return + } + t.FailNow() +} + +// IsDecreasing asserts that the collection is decreasing +// +// require.IsDecreasing(t, []int{2, 1, 0}) +// require.IsDecreasing(t, []float{2, 1}) +// require.IsDecreasing(t, []string{"b", "a"}) +func IsDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsDecreasing(t, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// IsDecreasingf asserts that the collection is decreasing +// +// require.IsDecreasingf(t, []int{2, 1, 0}, "error message %s", "formatted") +// require.IsDecreasingf(t, []float{2, 1}, "error message %s", "formatted") +// require.IsDecreasingf(t, []string{"b", "a"}, "error message %s", "formatted") +func IsDecreasingf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsDecreasingf(t, object, msg, args...) { + return + } + t.FailNow() +} + +// IsIncreasing asserts that the collection is increasing +// +// require.IsIncreasing(t, []int{1, 2, 3}) +// require.IsIncreasing(t, []float{1, 2}) +// require.IsIncreasing(t, []string{"a", "b"}) +func IsIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsIncreasing(t, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// IsIncreasingf asserts that the collection is increasing +// +// require.IsIncreasingf(t, []int{1, 2, 3}, "error message %s", "formatted") +// require.IsIncreasingf(t, []float{1, 2}, "error message %s", "formatted") +// require.IsIncreasingf(t, []string{"a", "b"}, "error message %s", "formatted") +func IsIncreasingf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsIncreasingf(t, object, msg, args...) { + return + } + t.FailNow() +} + +// IsNonDecreasing asserts that the collection is not decreasing +// +// require.IsNonDecreasing(t, []int{1, 1, 2}) +// require.IsNonDecreasing(t, []float{1, 2}) +// require.IsNonDecreasing(t, []string{"a", "b"}) +func IsNonDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsNonDecreasing(t, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// IsNonDecreasingf asserts that the collection is not decreasing +// +// require.IsNonDecreasingf(t, []int{1, 1, 2}, "error message %s", "formatted") +// require.IsNonDecreasingf(t, []float{1, 2}, "error message %s", "formatted") +// require.IsNonDecreasingf(t, []string{"a", "b"}, "error message %s", "formatted") +func IsNonDecreasingf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsNonDecreasingf(t, object, msg, args...) { + return + } + t.FailNow() +} + +// IsNonIncreasing asserts that the collection is not increasing +// +// require.IsNonIncreasing(t, []int{2, 1, 1}) +// require.IsNonIncreasing(t, []float{2, 1}) +// require.IsNonIncreasing(t, []string{"b", "a"}) +func IsNonIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsNonIncreasing(t, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// IsNonIncreasingf asserts that the collection is not increasing +// +// require.IsNonIncreasingf(t, []int{2, 1, 1}, "error message %s", "formatted") +// require.IsNonIncreasingf(t, []float{2, 1}, "error message %s", "formatted") +// require.IsNonIncreasingf(t, []string{"b", "a"}, "error message %s", "formatted") +func IsNonIncreasingf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsNonIncreasingf(t, object, msg, args...) { + return + } + t.FailNow() +} + +// IsNotType asserts that the specified objects are not of the same type. +// +// require.IsNotType(t, &NotMyStruct{}, &MyStruct{}) +func IsNotType(t TestingT, theType interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsNotType(t, theType, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// IsNotTypef asserts that the specified objects are not of the same type. +// +// require.IsNotTypef(t, &NotMyStruct{}, &MyStruct{}, "error message %s", "formatted") +func IsNotTypef(t TestingT, theType interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsNotTypef(t, theType, object, msg, args...) { + return + } + t.FailNow() +} + +// IsType asserts that the specified objects are of the same type. +// +// require.IsType(t, &MyStruct{}, &MyStruct{}) +func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsType(t, expectedType, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// IsTypef asserts that the specified objects are of the same type. +// +// require.IsTypef(t, &MyStruct{}, &MyStruct{}, "error message %s", "formatted") +func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsTypef(t, expectedType, object, msg, args...) { + return + } + t.FailNow() +} + +// JSONEq asserts that two JSON strings are equivalent. +// +// require.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) +func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.JSONEq(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// JSONEqf asserts that two JSON strings are equivalent. +// +// require.JSONEqf(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") +func JSONEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.JSONEqf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// Len asserts that the specified object has specific length. +// Len also fails if the object has a type that len() not accept. +// +// require.Len(t, mySlice, 3) +func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Len(t, object, length, msgAndArgs...) { + return + } + t.FailNow() +} + +// Lenf asserts that the specified object has specific length. +// Lenf also fails if the object has a type that len() not accept. +// +// require.Lenf(t, mySlice, 3, "error message %s", "formatted") +func Lenf(t TestingT, object interface{}, length int, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Lenf(t, object, length, msg, args...) { + return + } + t.FailNow() +} + +// Less asserts that the first element is less than the second +// +// require.Less(t, 1, 2) +// require.Less(t, float64(1), float64(2)) +// require.Less(t, "a", "b") +func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Less(t, e1, e2, msgAndArgs...) { + return + } + t.FailNow() +} + +// LessOrEqual asserts that the first element is less than or equal to the second +// +// require.LessOrEqual(t, 1, 2) +// require.LessOrEqual(t, 2, 2) +// require.LessOrEqual(t, "a", "b") +// require.LessOrEqual(t, "b", "b") +func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.LessOrEqual(t, e1, e2, msgAndArgs...) { + return + } + t.FailNow() +} + +// LessOrEqualf asserts that the first element is less than or equal to the second +// +// require.LessOrEqualf(t, 1, 2, "error message %s", "formatted") +// require.LessOrEqualf(t, 2, 2, "error message %s", "formatted") +// require.LessOrEqualf(t, "a", "b", "error message %s", "formatted") +// require.LessOrEqualf(t, "b", "b", "error message %s", "formatted") +func LessOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.LessOrEqualf(t, e1, e2, msg, args...) { + return + } + t.FailNow() +} + +// Lessf asserts that the first element is less than the second +// +// require.Lessf(t, 1, 2, "error message %s", "formatted") +// require.Lessf(t, float64(1), float64(2), "error message %s", "formatted") +// require.Lessf(t, "a", "b", "error message %s", "formatted") +func Lessf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Lessf(t, e1, e2, msg, args...) { + return + } + t.FailNow() +} + +// Negative asserts that the specified element is negative +// +// require.Negative(t, -1) +// require.Negative(t, -1.23) +func Negative(t TestingT, e interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Negative(t, e, msgAndArgs...) { + return + } + t.FailNow() +} + +// Negativef asserts that the specified element is negative +// +// require.Negativef(t, -1, "error message %s", "formatted") +// require.Negativef(t, -1.23, "error message %s", "formatted") +func Negativef(t TestingT, e interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Negativef(t, e, msg, args...) { + return + } + t.FailNow() +} + +// Never asserts that the given condition doesn't satisfy in waitFor time, +// periodically checking the target function each tick. +// +// require.Never(t, func() bool { return false; }, time.Second, 10*time.Millisecond) +func Never(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Never(t, condition, waitFor, tick, msgAndArgs...) { + return + } + t.FailNow() +} + +// Neverf asserts that the given condition doesn't satisfy in waitFor time, +// periodically checking the target function each tick. +// +// require.Neverf(t, func() bool { return false; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func Neverf(t TestingT, condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Neverf(t, condition, waitFor, tick, msg, args...) { + return + } + t.FailNow() +} + +// Nil asserts that the specified object is nil. +// +// require.Nil(t, err) +func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Nil(t, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// Nilf asserts that the specified object is nil. +// +// require.Nilf(t, err, "error message %s", "formatted") +func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Nilf(t, object, msg, args...) { + return + } + t.FailNow() +} + +// NoDirExists checks whether a directory does not exist in the given path. +// It fails if the path points to an existing _directory_ only. +func NoDirExists(t TestingT, path string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NoDirExists(t, path, msgAndArgs...) { + return + } + t.FailNow() +} + +// NoDirExistsf checks whether a directory does not exist in the given path. +// It fails if the path points to an existing _directory_ only. +func NoDirExistsf(t TestingT, path string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NoDirExistsf(t, path, msg, args...) { + return + } + t.FailNow() +} + +// NoError asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if require.NoError(t, err) { +// require.Equal(t, expectedObj, actualObj) +// } +func NoError(t TestingT, err error, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NoError(t, err, msgAndArgs...) { + return + } + t.FailNow() +} + +// NoErrorf asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if require.NoErrorf(t, err, "error message %s", "formatted") { +// require.Equal(t, expectedObj, actualObj) +// } +func NoErrorf(t TestingT, err error, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NoErrorf(t, err, msg, args...) { + return + } + t.FailNow() +} + +// NoFileExists checks whether a file does not exist in a given path. It fails +// if the path points to an existing _file_ only. +func NoFileExists(t TestingT, path string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NoFileExists(t, path, msgAndArgs...) { + return + } + t.FailNow() +} + +// NoFileExistsf checks whether a file does not exist in a given path. It fails +// if the path points to an existing _file_ only. +func NoFileExistsf(t TestingT, path string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NoFileExistsf(t, path, msg, args...) { + return + } + t.FailNow() +} + +// NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// require.NotContains(t, "Hello World", "Earth") +// require.NotContains(t, ["Hello", "World"], "Earth") +// require.NotContains(t, {"Hello": "World"}, "Earth") +func NotContains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotContains(t, s, contains, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotContainsf asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// require.NotContainsf(t, "Hello World", "Earth", "error message %s", "formatted") +// require.NotContainsf(t, ["Hello", "World"], "Earth", "error message %s", "formatted") +// require.NotContainsf(t, {"Hello": "World"}, "Earth", "error message %s", "formatted") +func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotContainsf(t, s, contains, msg, args...) { + return + } + t.FailNow() +} + +// NotElementsMatch asserts that the specified listA(array, slice...) is NOT equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should not match. +// This is an inverse of ElementsMatch. +// +// require.NotElementsMatch(t, [1, 1, 2, 3], [1, 1, 2, 3]) -> false +// +// require.NotElementsMatch(t, [1, 1, 2, 3], [1, 2, 3]) -> true +// +// require.NotElementsMatch(t, [1, 2, 3], [1, 2, 4]) -> true +func NotElementsMatch(t TestingT, listA interface{}, listB interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotElementsMatch(t, listA, listB, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotElementsMatchf asserts that the specified listA(array, slice...) is NOT equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should not match. +// This is an inverse of ElementsMatch. +// +// require.NotElementsMatchf(t, [1, 1, 2, 3], [1, 1, 2, 3], "error message %s", "formatted") -> false +// +// require.NotElementsMatchf(t, [1, 1, 2, 3], [1, 2, 3], "error message %s", "formatted") -> true +// +// require.NotElementsMatchf(t, [1, 2, 3], [1, 2, 4], "error message %s", "formatted") -> true +func NotElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotElementsMatchf(t, listA, listB, msg, args...) { + return + } + t.FailNow() +} + +// NotEmpty asserts that the specified object is NOT [Empty]. +// +// if require.NotEmpty(t, obj) { +// require.Equal(t, "two", obj[1]) +// } +func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotEmpty(t, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotEmptyf asserts that the specified object is NOT [Empty]. +// +// if require.NotEmptyf(t, obj, "error message %s", "formatted") { +// require.Equal(t, "two", obj[1]) +// } +func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotEmptyf(t, object, msg, args...) { + return + } + t.FailNow() +} + +// NotEqual asserts that the specified values are NOT equal. +// +// require.NotEqual(t, obj1, obj2) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). +func NotEqual(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotEqual(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotEqualValues asserts that two objects are not equal even when converted to the same type +// +// require.NotEqualValues(t, obj1, obj2) +func NotEqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotEqualValues(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotEqualValuesf asserts that two objects are not equal even when converted to the same type +// +// require.NotEqualValuesf(t, obj1, obj2, "error message %s", "formatted") +func NotEqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotEqualValuesf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// NotEqualf asserts that the specified values are NOT equal. +// +// require.NotEqualf(t, obj1, obj2, "error message %s", "formatted") +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). +func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotEqualf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// NotErrorAs asserts that none of the errors in err's chain matches target, +// but if so, sets target to that error value. +func NotErrorAs(t TestingT, err error, target interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotErrorAs(t, err, target, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotErrorAsf asserts that none of the errors in err's chain matches target, +// but if so, sets target to that error value. +func NotErrorAsf(t TestingT, err error, target interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotErrorAsf(t, err, target, msg, args...) { + return + } + t.FailNow() +} + +// NotErrorIs asserts that none of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func NotErrorIs(t TestingT, err error, target error, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotErrorIs(t, err, target, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotErrorIsf asserts that none of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func NotErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotErrorIsf(t, err, target, msg, args...) { + return + } + t.FailNow() +} + +// NotImplements asserts that an object does not implement the specified interface. +// +// require.NotImplements(t, (*MyInterface)(nil), new(MyObject)) +func NotImplements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotImplements(t, interfaceObject, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotImplementsf asserts that an object does not implement the specified interface. +// +// require.NotImplementsf(t, (*MyInterface)(nil), new(MyObject), "error message %s", "formatted") +func NotImplementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotImplementsf(t, interfaceObject, object, msg, args...) { + return + } + t.FailNow() +} + +// NotNil asserts that the specified object is not nil. +// +// require.NotNil(t, err) +func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotNil(t, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotNilf asserts that the specified object is not nil. +// +// require.NotNilf(t, err, "error message %s", "formatted") +func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotNilf(t, object, msg, args...) { + return + } + t.FailNow() +} + +// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// require.NotPanics(t, func(){ RemainCalm() }) +func NotPanics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotPanics(t, f, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// require.NotPanicsf(t, func(){ RemainCalm() }, "error message %s", "formatted") +func NotPanicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotPanicsf(t, f, msg, args...) { + return + } + t.FailNow() +} + +// NotRegexp asserts that a specified regexp does not match a string. +// +// require.NotRegexp(t, regexp.MustCompile("starts"), "it's starting") +// require.NotRegexp(t, "^start", "it's not starting") +func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotRegexp(t, rx, str, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotRegexpf asserts that a specified regexp does not match a string. +// +// require.NotRegexpf(t, regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") +// require.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted") +func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotRegexpf(t, rx, str, msg, args...) { + return + } + t.FailNow() +} + +// NotSame asserts that two pointers do not reference the same object. +// +// require.NotSame(t, ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func NotSame(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotSame(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotSamef asserts that two pointers do not reference the same object. +// +// require.NotSamef(t, ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func NotSamef(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotSamef(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// NotSubset asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// require.NotSubset(t, [1, 3, 4], [1, 2]) +// require.NotSubset(t, {"x": 1, "y": 2}, {"z": 3}) +// require.NotSubset(t, [1, 3, 4], {1: "one", 2: "two"}) +// require.NotSubset(t, {"x": 1, "y": 2}, ["z"]) +func NotSubset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotSubset(t, list, subset, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotSubsetf asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// require.NotSubsetf(t, [1, 3, 4], [1, 2], "error message %s", "formatted") +// require.NotSubsetf(t, {"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") +// require.NotSubsetf(t, [1, 3, 4], {1: "one", 2: "two"}, "error message %s", "formatted") +// require.NotSubsetf(t, {"x": 1, "y": 2}, ["z"], "error message %s", "formatted") +func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotSubsetf(t, list, subset, msg, args...) { + return + } + t.FailNow() +} + +// NotZero asserts that i is not the zero value for its type. +func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotZero(t, i, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotZerof asserts that i is not the zero value for its type. +func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotZerof(t, i, msg, args...) { + return + } + t.FailNow() +} + +// Panics asserts that the code inside the specified PanicTestFunc panics. +// +// require.Panics(t, func(){ GoCrazy() }) +func Panics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Panics(t, f, msgAndArgs...) { + return + } + t.FailNow() +} + +// PanicsWithError asserts that the code inside the specified PanicTestFunc +// panics, and that the recovered panic value is an error that satisfies the +// EqualError comparison. +// +// require.PanicsWithError(t, "crazy error", func(){ GoCrazy() }) +func PanicsWithError(t TestingT, errString string, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.PanicsWithError(t, errString, f, msgAndArgs...) { + return + } + t.FailNow() +} + +// PanicsWithErrorf asserts that the code inside the specified PanicTestFunc +// panics, and that the recovered panic value is an error that satisfies the +// EqualError comparison. +// +// require.PanicsWithErrorf(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") +func PanicsWithErrorf(t TestingT, errString string, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.PanicsWithErrorf(t, errString, f, msg, args...) { + return + } + t.FailNow() +} + +// PanicsWithValue asserts that the code inside the specified PanicTestFunc panics, and that +// the recovered panic value equals the expected panic value. +// +// require.PanicsWithValue(t, "crazy error", func(){ GoCrazy() }) +func PanicsWithValue(t TestingT, expected interface{}, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.PanicsWithValue(t, expected, f, msgAndArgs...) { + return + } + t.FailNow() +} + +// PanicsWithValuef asserts that the code inside the specified PanicTestFunc panics, and that +// the recovered panic value equals the expected panic value. +// +// require.PanicsWithValuef(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") +func PanicsWithValuef(t TestingT, expected interface{}, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.PanicsWithValuef(t, expected, f, msg, args...) { + return + } + t.FailNow() +} + +// Panicsf asserts that the code inside the specified PanicTestFunc panics. +// +// require.Panicsf(t, func(){ GoCrazy() }, "error message %s", "formatted") +func Panicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Panicsf(t, f, msg, args...) { + return + } + t.FailNow() +} + +// Positive asserts that the specified element is positive +// +// require.Positive(t, 1) +// require.Positive(t, 1.23) +func Positive(t TestingT, e interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Positive(t, e, msgAndArgs...) { + return + } + t.FailNow() +} + +// Positivef asserts that the specified element is positive +// +// require.Positivef(t, 1, "error message %s", "formatted") +// require.Positivef(t, 1.23, "error message %s", "formatted") +func Positivef(t TestingT, e interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Positivef(t, e, msg, args...) { + return + } + t.FailNow() +} + +// Regexp asserts that a specified regexp matches a string. +// +// require.Regexp(t, regexp.MustCompile("start"), "it's starting") +// require.Regexp(t, "start...$", "it's not starting") +func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Regexp(t, rx, str, msgAndArgs...) { + return + } + t.FailNow() +} + +// Regexpf asserts that a specified regexp matches a string. +// +// require.Regexpf(t, regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") +// require.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted") +func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Regexpf(t, rx, str, msg, args...) { + return + } + t.FailNow() +} + +// Same asserts that two pointers reference the same object. +// +// require.Same(t, ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func Same(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Same(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// Samef asserts that two pointers reference the same object. +// +// require.Samef(t, ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func Samef(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Samef(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// Subset asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// require.Subset(t, [1, 2, 3], [1, 2]) +// require.Subset(t, {"x": 1, "y": 2}, {"x": 1}) +// require.Subset(t, [1, 2, 3], {1: "one", 2: "two"}) +// require.Subset(t, {"x": 1, "y": 2}, ["x"]) +func Subset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Subset(t, list, subset, msgAndArgs...) { + return + } + t.FailNow() +} + +// Subsetf asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// require.Subsetf(t, [1, 2, 3], [1, 2], "error message %s", "formatted") +// require.Subsetf(t, {"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") +// require.Subsetf(t, [1, 2, 3], {1: "one", 2: "two"}, "error message %s", "formatted") +// require.Subsetf(t, {"x": 1, "y": 2}, ["x"], "error message %s", "formatted") +func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Subsetf(t, list, subset, msg, args...) { + return + } + t.FailNow() +} + +// True asserts that the specified value is true. +// +// require.True(t, myBool) +func True(t TestingT, value bool, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.True(t, value, msgAndArgs...) { + return + } + t.FailNow() +} + +// Truef asserts that the specified value is true. +// +// require.Truef(t, myBool, "error message %s", "formatted") +func Truef(t TestingT, value bool, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Truef(t, value, msg, args...) { + return + } + t.FailNow() +} + +// WithinDuration asserts that the two times are within duration delta of each other. +// +// require.WithinDuration(t, time.Now(), time.Now(), 10*time.Second) +func WithinDuration(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.WithinDuration(t, expected, actual, delta, msgAndArgs...) { + return + } + t.FailNow() +} + +// WithinDurationf asserts that the two times are within duration delta of each other. +// +// require.WithinDurationf(t, time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") +func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.WithinDurationf(t, expected, actual, delta, msg, args...) { + return + } + t.FailNow() +} + +// WithinRange asserts that a time is within a time range (inclusive). +// +// require.WithinRange(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) +func WithinRange(t TestingT, actual time.Time, start time.Time, end time.Time, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.WithinRange(t, actual, start, end, msgAndArgs...) { + return + } + t.FailNow() +} + +// WithinRangef asserts that a time is within a time range (inclusive). +// +// require.WithinRangef(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") +func WithinRangef(t TestingT, actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.WithinRangef(t, actual, start, end, msg, args...) { + return + } + t.FailNow() +} + +// YAMLEq asserts that two YAML strings are equivalent. +func YAMLEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.YAMLEq(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// YAMLEqf asserts that two YAML strings are equivalent. +func YAMLEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.YAMLEqf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + +// Zero asserts that i is the zero value for its type. +func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Zero(t, i, msgAndArgs...) { + return + } + t.FailNow() +} + +// Zerof asserts that i is the zero value for its type. +func Zerof(t TestingT, i interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.Zerof(t, i, msg, args...) { + return + } + t.FailNow() +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/require/require.go.tmpl b/scraper-go/vendor/github.com/stretchr/testify/require/require.go.tmpl new file mode 100644 index 00000000..8b328368 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/require/require.go.tmpl @@ -0,0 +1,6 @@ +{{ replace .Comment "assert." "require."}} +func {{.DocInfo.Name}}(t TestingT, {{.Params}}) { + if h, ok := t.(tHelper); ok { h.Helper() } + if assert.{{.DocInfo.Name}}(t, {{.ForwardedParams}}) { return } + t.FailNow() +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/require/require_forward.go b/scraper-go/vendor/github.com/stretchr/testify/require/require_forward.go new file mode 100644 index 00000000..e6f7e944 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/require/require_forward.go @@ -0,0 +1,1724 @@ +// Code generated with github.com/stretchr/testify/_codegen; DO NOT EDIT. + +package require + +import ( + assert "github.com/stretchr/testify/assert" + http "net/http" + url "net/url" + time "time" +) + +// Condition uses a Comparison to assert a complex condition. +func (a *Assertions) Condition(comp assert.Comparison, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Condition(a.t, comp, msgAndArgs...) +} + +// Conditionf uses a Comparison to assert a complex condition. +func (a *Assertions) Conditionf(comp assert.Comparison, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Conditionf(a.t, comp, msg, args...) +} + +// Contains asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// a.Contains("Hello World", "World") +// a.Contains(["Hello", "World"], "World") +// a.Contains({"Hello": "World"}, "Hello") +func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Contains(a.t, s, contains, msgAndArgs...) +} + +// Containsf asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// a.Containsf("Hello World", "World", "error message %s", "formatted") +// a.Containsf(["Hello", "World"], "World", "error message %s", "formatted") +// a.Containsf({"Hello": "World"}, "Hello", "error message %s", "formatted") +func (a *Assertions) Containsf(s interface{}, contains interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Containsf(a.t, s, contains, msg, args...) +} + +// DirExists checks whether a directory exists in the given path. It also fails +// if the path is a file rather a directory or there is an error checking whether it exists. +func (a *Assertions) DirExists(path string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + DirExists(a.t, path, msgAndArgs...) +} + +// DirExistsf checks whether a directory exists in the given path. It also fails +// if the path is a file rather a directory or there is an error checking whether it exists. +func (a *Assertions) DirExistsf(path string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + DirExistsf(a.t, path, msg, args...) +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// a.ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]) +func (a *Assertions) ElementsMatch(listA interface{}, listB interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ElementsMatch(a.t, listA, listB, msgAndArgs...) +} + +// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// a.ElementsMatchf([1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") +func (a *Assertions) ElementsMatchf(listA interface{}, listB interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ElementsMatchf(a.t, listA, listB, msg, args...) +} + +// Empty asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". +// +// a.Empty(obj) +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value +func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Empty(a.t, object, msgAndArgs...) +} + +// Emptyf asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". +// +// a.Emptyf(obj, "error message %s", "formatted") +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value +func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Emptyf(a.t, object, msg, args...) +} + +// Equal asserts that two objects are equal. +// +// a.Equal(123, 123) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Equal(a.t, expected, actual, msgAndArgs...) +} + +// EqualError asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// a.EqualError(err, expectedErrorString) +func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + EqualError(a.t, theError, errString, msgAndArgs...) +} + +// EqualErrorf asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// a.EqualErrorf(err, expectedErrorString, "error message %s", "formatted") +func (a *Assertions) EqualErrorf(theError error, errString string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + EqualErrorf(a.t, theError, errString, msg, args...) +} + +// EqualExportedValues asserts that the types of two objects are equal and their public +// fields are also equal. This is useful for comparing structs that have private fields +// that could potentially differ. +// +// type S struct { +// Exported int +// notExported int +// } +// a.EqualExportedValues(S{1, 2}, S{1, 3}) => true +// a.EqualExportedValues(S{1, 2}, S{2, 3}) => false +func (a *Assertions) EqualExportedValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + EqualExportedValues(a.t, expected, actual, msgAndArgs...) +} + +// EqualExportedValuesf asserts that the types of two objects are equal and their public +// fields are also equal. This is useful for comparing structs that have private fields +// that could potentially differ. +// +// type S struct { +// Exported int +// notExported int +// } +// a.EqualExportedValuesf(S{1, 2}, S{1, 3}, "error message %s", "formatted") => true +// a.EqualExportedValuesf(S{1, 2}, S{2, 3}, "error message %s", "formatted") => false +func (a *Assertions) EqualExportedValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + EqualExportedValuesf(a.t, expected, actual, msg, args...) +} + +// EqualValues asserts that two objects are equal or convertible to the larger +// type and equal. +// +// a.EqualValues(uint32(123), int32(123)) +func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + EqualValues(a.t, expected, actual, msgAndArgs...) +} + +// EqualValuesf asserts that two objects are equal or convertible to the larger +// type and equal. +// +// a.EqualValuesf(uint32(123), int32(123), "error message %s", "formatted") +func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + EqualValuesf(a.t, expected, actual, msg, args...) +} + +// Equalf asserts that two objects are equal. +// +// a.Equalf(123, 123, "error message %s", "formatted") +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Equalf(a.t, expected, actual, msg, args...) +} + +// Error asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// a.Error(err) +func (a *Assertions) Error(err error, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Error(a.t, err, msgAndArgs...) +} + +// ErrorAs asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. +// This is a wrapper for errors.As. +func (a *Assertions) ErrorAs(err error, target interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ErrorAs(a.t, err, target, msgAndArgs...) +} + +// ErrorAsf asserts that at least one of the errors in err's chain matches target, and if so, sets target to that error value. +// This is a wrapper for errors.As. +func (a *Assertions) ErrorAsf(err error, target interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ErrorAsf(a.t, err, target, msg, args...) +} + +// ErrorContains asserts that a function returned an error (i.e. not `nil`) +// and that the error contains the specified substring. +// +// actualObj, err := SomeFunction() +// a.ErrorContains(err, expectedErrorSubString) +func (a *Assertions) ErrorContains(theError error, contains string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ErrorContains(a.t, theError, contains, msgAndArgs...) +} + +// ErrorContainsf asserts that a function returned an error (i.e. not `nil`) +// and that the error contains the specified substring. +// +// actualObj, err := SomeFunction() +// a.ErrorContainsf(err, expectedErrorSubString, "error message %s", "formatted") +func (a *Assertions) ErrorContainsf(theError error, contains string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ErrorContainsf(a.t, theError, contains, msg, args...) +} + +// ErrorIs asserts that at least one of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func (a *Assertions) ErrorIs(err error, target error, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ErrorIs(a.t, err, target, msgAndArgs...) +} + +// ErrorIsf asserts that at least one of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func (a *Assertions) ErrorIsf(err error, target error, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ErrorIsf(a.t, err, target, msg, args...) +} + +// Errorf asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// a.Errorf(err, "error message %s", "formatted") +func (a *Assertions) Errorf(err error, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Errorf(a.t, err, msg, args...) +} + +// Eventually asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// a.Eventually(func() bool { return true; }, time.Second, 10*time.Millisecond) +func (a *Assertions) Eventually(condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Eventually(a.t, condition, waitFor, tick, msgAndArgs...) +} + +// EventuallyWithT asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. In contrast to Eventually, +// it supplies a CollectT to the condition function, so that the condition +// function can use the CollectT to call other assertions. +// The condition is considered "met" if no errors are raised in a tick. +// The supplied CollectT collects all errors from one tick (if there are any). +// If the condition is not met before waitFor, the collected errors of +// the last tick are copied to t. +// +// externalValue := false +// go func() { +// time.Sleep(8*time.Second) +// externalValue = true +// }() +// a.EventuallyWithT(func(c *assert.CollectT) { +// // add assertions as needed; any assertion failure will fail the current tick +// assert.True(c, externalValue, "expected 'externalValue' to be true") +// }, 10*time.Second, 1*time.Second, "external state has not changed to 'true'; still false") +func (a *Assertions) EventuallyWithT(condition func(collect *assert.CollectT), waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + EventuallyWithT(a.t, condition, waitFor, tick, msgAndArgs...) +} + +// EventuallyWithTf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. In contrast to Eventually, +// it supplies a CollectT to the condition function, so that the condition +// function can use the CollectT to call other assertions. +// The condition is considered "met" if no errors are raised in a tick. +// The supplied CollectT collects all errors from one tick (if there are any). +// If the condition is not met before waitFor, the collected errors of +// the last tick are copied to t. +// +// externalValue := false +// go func() { +// time.Sleep(8*time.Second) +// externalValue = true +// }() +// a.EventuallyWithTf(func(c *assert.CollectT, "error message %s", "formatted") { +// // add assertions as needed; any assertion failure will fail the current tick +// assert.True(c, externalValue, "expected 'externalValue' to be true") +// }, 10*time.Second, 1*time.Second, "external state has not changed to 'true'; still false") +func (a *Assertions) EventuallyWithTf(condition func(collect *assert.CollectT), waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + EventuallyWithTf(a.t, condition, waitFor, tick, msg, args...) +} + +// Eventuallyf asserts that given condition will be met in waitFor time, +// periodically checking target function each tick. +// +// a.Eventuallyf(func() bool { return true; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func (a *Assertions) Eventuallyf(condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Eventuallyf(a.t, condition, waitFor, tick, msg, args...) +} + +// Exactly asserts that two objects are equal in value and type. +// +// a.Exactly(int32(123), int64(123)) +func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Exactly(a.t, expected, actual, msgAndArgs...) +} + +// Exactlyf asserts that two objects are equal in value and type. +// +// a.Exactlyf(int32(123), int64(123), "error message %s", "formatted") +func (a *Assertions) Exactlyf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Exactlyf(a.t, expected, actual, msg, args...) +} + +// Fail reports a failure through +func (a *Assertions) Fail(failureMessage string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Fail(a.t, failureMessage, msgAndArgs...) +} + +// FailNow fails test +func (a *Assertions) FailNow(failureMessage string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + FailNow(a.t, failureMessage, msgAndArgs...) +} + +// FailNowf fails test +func (a *Assertions) FailNowf(failureMessage string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + FailNowf(a.t, failureMessage, msg, args...) +} + +// Failf reports a failure through +func (a *Assertions) Failf(failureMessage string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Failf(a.t, failureMessage, msg, args...) +} + +// False asserts that the specified value is false. +// +// a.False(myBool) +func (a *Assertions) False(value bool, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + False(a.t, value, msgAndArgs...) +} + +// Falsef asserts that the specified value is false. +// +// a.Falsef(myBool, "error message %s", "formatted") +func (a *Assertions) Falsef(value bool, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Falsef(a.t, value, msg, args...) +} + +// FileExists checks whether a file exists in the given path. It also fails if +// the path points to a directory or there is an error when trying to check the file. +func (a *Assertions) FileExists(path string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + FileExists(a.t, path, msgAndArgs...) +} + +// FileExistsf checks whether a file exists in the given path. It also fails if +// the path points to a directory or there is an error when trying to check the file. +func (a *Assertions) FileExistsf(path string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + FileExistsf(a.t, path, msg, args...) +} + +// Greater asserts that the first element is greater than the second +// +// a.Greater(2, 1) +// a.Greater(float64(2), float64(1)) +// a.Greater("b", "a") +func (a *Assertions) Greater(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Greater(a.t, e1, e2, msgAndArgs...) +} + +// GreaterOrEqual asserts that the first element is greater than or equal to the second +// +// a.GreaterOrEqual(2, 1) +// a.GreaterOrEqual(2, 2) +// a.GreaterOrEqual("b", "a") +// a.GreaterOrEqual("b", "b") +func (a *Assertions) GreaterOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + GreaterOrEqual(a.t, e1, e2, msgAndArgs...) +} + +// GreaterOrEqualf asserts that the first element is greater than or equal to the second +// +// a.GreaterOrEqualf(2, 1, "error message %s", "formatted") +// a.GreaterOrEqualf(2, 2, "error message %s", "formatted") +// a.GreaterOrEqualf("b", "a", "error message %s", "formatted") +// a.GreaterOrEqualf("b", "b", "error message %s", "formatted") +func (a *Assertions) GreaterOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + GreaterOrEqualf(a.t, e1, e2, msg, args...) +} + +// Greaterf asserts that the first element is greater than the second +// +// a.Greaterf(2, 1, "error message %s", "formatted") +// a.Greaterf(float64(2), float64(1), "error message %s", "formatted") +// a.Greaterf("b", "a", "error message %s", "formatted") +func (a *Assertions) Greaterf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Greaterf(a.t, e1, e2, msg, args...) +} + +// HTTPBodyContains asserts that a specified handler returns a +// body that contains a string. +// +// a.HTTPBodyContains(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPBodyContains(a.t, handler, method, url, values, str, msgAndArgs...) +} + +// HTTPBodyContainsf asserts that a specified handler returns a +// body that contains a string. +// +// a.HTTPBodyContainsf(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPBodyContainsf(a.t, handler, method, url, values, str, msg, args...) +} + +// HTTPBodyNotContains asserts that a specified handler returns a +// body that does not contain a string. +// +// a.HTTPBodyNotContains(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPBodyNotContains(a.t, handler, method, url, values, str, msgAndArgs...) +} + +// HTTPBodyNotContainsf asserts that a specified handler returns a +// body that does not contain a string. +// +// a.HTTPBodyNotContainsf(myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPBodyNotContainsf(a.t, handler, method, url, values, str, msg, args...) +} + +// HTTPError asserts that a specified handler returns an error status code. +// +// a.HTTPError(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPError(a.t, handler, method, url, values, msgAndArgs...) +} + +// HTTPErrorf asserts that a specified handler returns an error status code. +// +// a.HTTPErrorf(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPErrorf(a.t, handler, method, url, values, msg, args...) +} + +// HTTPRedirect asserts that a specified handler returns a redirect status code. +// +// a.HTTPRedirect(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPRedirect(a.t, handler, method, url, values, msgAndArgs...) +} + +// HTTPRedirectf asserts that a specified handler returns a redirect status code. +// +// a.HTTPRedirectf(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPRedirectf(a.t, handler, method, url, values, msg, args...) +} + +// HTTPStatusCode asserts that a specified handler returns a specified status code. +// +// a.HTTPStatusCode(myHandler, "GET", "/notImplemented", nil, 501) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPStatusCode(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPStatusCode(a.t, handler, method, url, values, statuscode, msgAndArgs...) +} + +// HTTPStatusCodef asserts that a specified handler returns a specified status code. +// +// a.HTTPStatusCodef(myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPStatusCodef(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPStatusCodef(a.t, handler, method, url, values, statuscode, msg, args...) +} + +// HTTPSuccess asserts that a specified handler returns a success status code. +// +// a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPSuccess(a.t, handler, method, url, values, msgAndArgs...) +} + +// HTTPSuccessf asserts that a specified handler returns a success status code. +// +// a.HTTPSuccessf(myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPSuccessf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPSuccessf(a.t, handler, method, url, values, msg, args...) +} + +// Implements asserts that an object is implemented by the specified interface. +// +// a.Implements((*MyInterface)(nil), new(MyObject)) +func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Implements(a.t, interfaceObject, object, msgAndArgs...) +} + +// Implementsf asserts that an object is implemented by the specified interface. +// +// a.Implementsf((*MyInterface)(nil), new(MyObject), "error message %s", "formatted") +func (a *Assertions) Implementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Implementsf(a.t, interfaceObject, object, msg, args...) +} + +// InDelta asserts that the two numerals are within delta of each other. +// +// a.InDelta(math.Pi, 22/7.0, 0.01) +func (a *Assertions) InDelta(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InDelta(a.t, expected, actual, delta, msgAndArgs...) +} + +// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func (a *Assertions) InDeltaMapValues(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InDeltaMapValues(a.t, expected, actual, delta, msgAndArgs...) +} + +// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func (a *Assertions) InDeltaMapValuesf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InDeltaMapValuesf(a.t, expected, actual, delta, msg, args...) +} + +// InDeltaSlice is the same as InDelta, except it compares two slices. +func (a *Assertions) InDeltaSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InDeltaSlice(a.t, expected, actual, delta, msgAndArgs...) +} + +// InDeltaSlicef is the same as InDelta, except it compares two slices. +func (a *Assertions) InDeltaSlicef(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InDeltaSlicef(a.t, expected, actual, delta, msg, args...) +} + +// InDeltaf asserts that the two numerals are within delta of each other. +// +// a.InDeltaf(math.Pi, 22/7.0, 0.01, "error message %s", "formatted") +func (a *Assertions) InDeltaf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InDeltaf(a.t, expected, actual, delta, msg, args...) +} + +// InEpsilon asserts that expected and actual have a relative error less than epsilon +func (a *Assertions) InEpsilon(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InEpsilon(a.t, expected, actual, epsilon, msgAndArgs...) +} + +// InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. +func (a *Assertions) InEpsilonSlice(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InEpsilonSlice(a.t, expected, actual, epsilon, msgAndArgs...) +} + +// InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. +func (a *Assertions) InEpsilonSlicef(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InEpsilonSlicef(a.t, expected, actual, epsilon, msg, args...) +} + +// InEpsilonf asserts that expected and actual have a relative error less than epsilon +func (a *Assertions) InEpsilonf(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InEpsilonf(a.t, expected, actual, epsilon, msg, args...) +} + +// IsDecreasing asserts that the collection is decreasing +// +// a.IsDecreasing([]int{2, 1, 0}) +// a.IsDecreasing([]float{2, 1}) +// a.IsDecreasing([]string{"b", "a"}) +func (a *Assertions) IsDecreasing(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsDecreasing(a.t, object, msgAndArgs...) +} + +// IsDecreasingf asserts that the collection is decreasing +// +// a.IsDecreasingf([]int{2, 1, 0}, "error message %s", "formatted") +// a.IsDecreasingf([]float{2, 1}, "error message %s", "formatted") +// a.IsDecreasingf([]string{"b", "a"}, "error message %s", "formatted") +func (a *Assertions) IsDecreasingf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsDecreasingf(a.t, object, msg, args...) +} + +// IsIncreasing asserts that the collection is increasing +// +// a.IsIncreasing([]int{1, 2, 3}) +// a.IsIncreasing([]float{1, 2}) +// a.IsIncreasing([]string{"a", "b"}) +func (a *Assertions) IsIncreasing(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsIncreasing(a.t, object, msgAndArgs...) +} + +// IsIncreasingf asserts that the collection is increasing +// +// a.IsIncreasingf([]int{1, 2, 3}, "error message %s", "formatted") +// a.IsIncreasingf([]float{1, 2}, "error message %s", "formatted") +// a.IsIncreasingf([]string{"a", "b"}, "error message %s", "formatted") +func (a *Assertions) IsIncreasingf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsIncreasingf(a.t, object, msg, args...) +} + +// IsNonDecreasing asserts that the collection is not decreasing +// +// a.IsNonDecreasing([]int{1, 1, 2}) +// a.IsNonDecreasing([]float{1, 2}) +// a.IsNonDecreasing([]string{"a", "b"}) +func (a *Assertions) IsNonDecreasing(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsNonDecreasing(a.t, object, msgAndArgs...) +} + +// IsNonDecreasingf asserts that the collection is not decreasing +// +// a.IsNonDecreasingf([]int{1, 1, 2}, "error message %s", "formatted") +// a.IsNonDecreasingf([]float{1, 2}, "error message %s", "formatted") +// a.IsNonDecreasingf([]string{"a", "b"}, "error message %s", "formatted") +func (a *Assertions) IsNonDecreasingf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsNonDecreasingf(a.t, object, msg, args...) +} + +// IsNonIncreasing asserts that the collection is not increasing +// +// a.IsNonIncreasing([]int{2, 1, 1}) +// a.IsNonIncreasing([]float{2, 1}) +// a.IsNonIncreasing([]string{"b", "a"}) +func (a *Assertions) IsNonIncreasing(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsNonIncreasing(a.t, object, msgAndArgs...) +} + +// IsNonIncreasingf asserts that the collection is not increasing +// +// a.IsNonIncreasingf([]int{2, 1, 1}, "error message %s", "formatted") +// a.IsNonIncreasingf([]float{2, 1}, "error message %s", "formatted") +// a.IsNonIncreasingf([]string{"b", "a"}, "error message %s", "formatted") +func (a *Assertions) IsNonIncreasingf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsNonIncreasingf(a.t, object, msg, args...) +} + +// IsNotType asserts that the specified objects are not of the same type. +// +// a.IsNotType(&NotMyStruct{}, &MyStruct{}) +func (a *Assertions) IsNotType(theType interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsNotType(a.t, theType, object, msgAndArgs...) +} + +// IsNotTypef asserts that the specified objects are not of the same type. +// +// a.IsNotTypef(&NotMyStruct{}, &MyStruct{}, "error message %s", "formatted") +func (a *Assertions) IsNotTypef(theType interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsNotTypef(a.t, theType, object, msg, args...) +} + +// IsType asserts that the specified objects are of the same type. +// +// a.IsType(&MyStruct{}, &MyStruct{}) +func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsType(a.t, expectedType, object, msgAndArgs...) +} + +// IsTypef asserts that the specified objects are of the same type. +// +// a.IsTypef(&MyStruct{}, &MyStruct{}, "error message %s", "formatted") +func (a *Assertions) IsTypef(expectedType interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsTypef(a.t, expectedType, object, msg, args...) +} + +// JSONEq asserts that two JSON strings are equivalent. +// +// a.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) +func (a *Assertions) JSONEq(expected string, actual string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + JSONEq(a.t, expected, actual, msgAndArgs...) +} + +// JSONEqf asserts that two JSON strings are equivalent. +// +// a.JSONEqf(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") +func (a *Assertions) JSONEqf(expected string, actual string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + JSONEqf(a.t, expected, actual, msg, args...) +} + +// Len asserts that the specified object has specific length. +// Len also fails if the object has a type that len() not accept. +// +// a.Len(mySlice, 3) +func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Len(a.t, object, length, msgAndArgs...) +} + +// Lenf asserts that the specified object has specific length. +// Lenf also fails if the object has a type that len() not accept. +// +// a.Lenf(mySlice, 3, "error message %s", "formatted") +func (a *Assertions) Lenf(object interface{}, length int, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Lenf(a.t, object, length, msg, args...) +} + +// Less asserts that the first element is less than the second +// +// a.Less(1, 2) +// a.Less(float64(1), float64(2)) +// a.Less("a", "b") +func (a *Assertions) Less(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Less(a.t, e1, e2, msgAndArgs...) +} + +// LessOrEqual asserts that the first element is less than or equal to the second +// +// a.LessOrEqual(1, 2) +// a.LessOrEqual(2, 2) +// a.LessOrEqual("a", "b") +// a.LessOrEqual("b", "b") +func (a *Assertions) LessOrEqual(e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + LessOrEqual(a.t, e1, e2, msgAndArgs...) +} + +// LessOrEqualf asserts that the first element is less than or equal to the second +// +// a.LessOrEqualf(1, 2, "error message %s", "formatted") +// a.LessOrEqualf(2, 2, "error message %s", "formatted") +// a.LessOrEqualf("a", "b", "error message %s", "formatted") +// a.LessOrEqualf("b", "b", "error message %s", "formatted") +func (a *Assertions) LessOrEqualf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + LessOrEqualf(a.t, e1, e2, msg, args...) +} + +// Lessf asserts that the first element is less than the second +// +// a.Lessf(1, 2, "error message %s", "formatted") +// a.Lessf(float64(1), float64(2), "error message %s", "formatted") +// a.Lessf("a", "b", "error message %s", "formatted") +func (a *Assertions) Lessf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Lessf(a.t, e1, e2, msg, args...) +} + +// Negative asserts that the specified element is negative +// +// a.Negative(-1) +// a.Negative(-1.23) +func (a *Assertions) Negative(e interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Negative(a.t, e, msgAndArgs...) +} + +// Negativef asserts that the specified element is negative +// +// a.Negativef(-1, "error message %s", "formatted") +// a.Negativef(-1.23, "error message %s", "formatted") +func (a *Assertions) Negativef(e interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Negativef(a.t, e, msg, args...) +} + +// Never asserts that the given condition doesn't satisfy in waitFor time, +// periodically checking the target function each tick. +// +// a.Never(func() bool { return false; }, time.Second, 10*time.Millisecond) +func (a *Assertions) Never(condition func() bool, waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Never(a.t, condition, waitFor, tick, msgAndArgs...) +} + +// Neverf asserts that the given condition doesn't satisfy in waitFor time, +// periodically checking the target function each tick. +// +// a.Neverf(func() bool { return false; }, time.Second, 10*time.Millisecond, "error message %s", "formatted") +func (a *Assertions) Neverf(condition func() bool, waitFor time.Duration, tick time.Duration, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Neverf(a.t, condition, waitFor, tick, msg, args...) +} + +// Nil asserts that the specified object is nil. +// +// a.Nil(err) +func (a *Assertions) Nil(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Nil(a.t, object, msgAndArgs...) +} + +// Nilf asserts that the specified object is nil. +// +// a.Nilf(err, "error message %s", "formatted") +func (a *Assertions) Nilf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Nilf(a.t, object, msg, args...) +} + +// NoDirExists checks whether a directory does not exist in the given path. +// It fails if the path points to an existing _directory_ only. +func (a *Assertions) NoDirExists(path string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NoDirExists(a.t, path, msgAndArgs...) +} + +// NoDirExistsf checks whether a directory does not exist in the given path. +// It fails if the path points to an existing _directory_ only. +func (a *Assertions) NoDirExistsf(path string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NoDirExistsf(a.t, path, msg, args...) +} + +// NoError asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if a.NoError(err) { +// assert.Equal(t, expectedObj, actualObj) +// } +func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NoError(a.t, err, msgAndArgs...) +} + +// NoErrorf asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if a.NoErrorf(err, "error message %s", "formatted") { +// assert.Equal(t, expectedObj, actualObj) +// } +func (a *Assertions) NoErrorf(err error, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NoErrorf(a.t, err, msg, args...) +} + +// NoFileExists checks whether a file does not exist in a given path. It fails +// if the path points to an existing _file_ only. +func (a *Assertions) NoFileExists(path string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NoFileExists(a.t, path, msgAndArgs...) +} + +// NoFileExistsf checks whether a file does not exist in a given path. It fails +// if the path points to an existing _file_ only. +func (a *Assertions) NoFileExistsf(path string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NoFileExistsf(a.t, path, msg, args...) +} + +// NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// a.NotContains("Hello World", "Earth") +// a.NotContains(["Hello", "World"], "Earth") +// a.NotContains({"Hello": "World"}, "Earth") +func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotContains(a.t, s, contains, msgAndArgs...) +} + +// NotContainsf asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// a.NotContainsf("Hello World", "Earth", "error message %s", "formatted") +// a.NotContainsf(["Hello", "World"], "Earth", "error message %s", "formatted") +// a.NotContainsf({"Hello": "World"}, "Earth", "error message %s", "formatted") +func (a *Assertions) NotContainsf(s interface{}, contains interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotContainsf(a.t, s, contains, msg, args...) +} + +// NotElementsMatch asserts that the specified listA(array, slice...) is NOT equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should not match. +// This is an inverse of ElementsMatch. +// +// a.NotElementsMatch([1, 1, 2, 3], [1, 1, 2, 3]) -> false +// +// a.NotElementsMatch([1, 1, 2, 3], [1, 2, 3]) -> true +// +// a.NotElementsMatch([1, 2, 3], [1, 2, 4]) -> true +func (a *Assertions) NotElementsMatch(listA interface{}, listB interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotElementsMatch(a.t, listA, listB, msgAndArgs...) +} + +// NotElementsMatchf asserts that the specified listA(array, slice...) is NOT equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should not match. +// This is an inverse of ElementsMatch. +// +// a.NotElementsMatchf([1, 1, 2, 3], [1, 1, 2, 3], "error message %s", "formatted") -> false +// +// a.NotElementsMatchf([1, 1, 2, 3], [1, 2, 3], "error message %s", "formatted") -> true +// +// a.NotElementsMatchf([1, 2, 3], [1, 2, 4], "error message %s", "formatted") -> true +func (a *Assertions) NotElementsMatchf(listA interface{}, listB interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotElementsMatchf(a.t, listA, listB, msg, args...) +} + +// NotEmpty asserts that the specified object is NOT [Empty]. +// +// if a.NotEmpty(obj) { +// assert.Equal(t, "two", obj[1]) +// } +func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotEmpty(a.t, object, msgAndArgs...) +} + +// NotEmptyf asserts that the specified object is NOT [Empty]. +// +// if a.NotEmptyf(obj, "error message %s", "formatted") { +// assert.Equal(t, "two", obj[1]) +// } +func (a *Assertions) NotEmptyf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotEmptyf(a.t, object, msg, args...) +} + +// NotEqual asserts that the specified values are NOT equal. +// +// a.NotEqual(obj1, obj2) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). +func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotEqual(a.t, expected, actual, msgAndArgs...) +} + +// NotEqualValues asserts that two objects are not equal even when converted to the same type +// +// a.NotEqualValues(obj1, obj2) +func (a *Assertions) NotEqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotEqualValues(a.t, expected, actual, msgAndArgs...) +} + +// NotEqualValuesf asserts that two objects are not equal even when converted to the same type +// +// a.NotEqualValuesf(obj1, obj2, "error message %s", "formatted") +func (a *Assertions) NotEqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotEqualValuesf(a.t, expected, actual, msg, args...) +} + +// NotEqualf asserts that the specified values are NOT equal. +// +// a.NotEqualf(obj1, obj2, "error message %s", "formatted") +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). +func (a *Assertions) NotEqualf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotEqualf(a.t, expected, actual, msg, args...) +} + +// NotErrorAs asserts that none of the errors in err's chain matches target, +// but if so, sets target to that error value. +func (a *Assertions) NotErrorAs(err error, target interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotErrorAs(a.t, err, target, msgAndArgs...) +} + +// NotErrorAsf asserts that none of the errors in err's chain matches target, +// but if so, sets target to that error value. +func (a *Assertions) NotErrorAsf(err error, target interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotErrorAsf(a.t, err, target, msg, args...) +} + +// NotErrorIs asserts that none of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func (a *Assertions) NotErrorIs(err error, target error, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotErrorIs(a.t, err, target, msgAndArgs...) +} + +// NotErrorIsf asserts that none of the errors in err's chain matches target. +// This is a wrapper for errors.Is. +func (a *Assertions) NotErrorIsf(err error, target error, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotErrorIsf(a.t, err, target, msg, args...) +} + +// NotImplements asserts that an object does not implement the specified interface. +// +// a.NotImplements((*MyInterface)(nil), new(MyObject)) +func (a *Assertions) NotImplements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotImplements(a.t, interfaceObject, object, msgAndArgs...) +} + +// NotImplementsf asserts that an object does not implement the specified interface. +// +// a.NotImplementsf((*MyInterface)(nil), new(MyObject), "error message %s", "formatted") +func (a *Assertions) NotImplementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotImplementsf(a.t, interfaceObject, object, msg, args...) +} + +// NotNil asserts that the specified object is not nil. +// +// a.NotNil(err) +func (a *Assertions) NotNil(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotNil(a.t, object, msgAndArgs...) +} + +// NotNilf asserts that the specified object is not nil. +// +// a.NotNilf(err, "error message %s", "formatted") +func (a *Assertions) NotNilf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotNilf(a.t, object, msg, args...) +} + +// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// a.NotPanics(func(){ RemainCalm() }) +func (a *Assertions) NotPanics(f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotPanics(a.t, f, msgAndArgs...) +} + +// NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// a.NotPanicsf(func(){ RemainCalm() }, "error message %s", "formatted") +func (a *Assertions) NotPanicsf(f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotPanicsf(a.t, f, msg, args...) +} + +// NotRegexp asserts that a specified regexp does not match a string. +// +// a.NotRegexp(regexp.MustCompile("starts"), "it's starting") +// a.NotRegexp("^start", "it's not starting") +func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotRegexp(a.t, rx, str, msgAndArgs...) +} + +// NotRegexpf asserts that a specified regexp does not match a string. +// +// a.NotRegexpf(regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") +// a.NotRegexpf("^start", "it's not starting", "error message %s", "formatted") +func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotRegexpf(a.t, rx, str, msg, args...) +} + +// NotSame asserts that two pointers do not reference the same object. +// +// a.NotSame(ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) NotSame(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotSame(a.t, expected, actual, msgAndArgs...) +} + +// NotSamef asserts that two pointers do not reference the same object. +// +// a.NotSamef(ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) NotSamef(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotSamef(a.t, expected, actual, msg, args...) +} + +// NotSubset asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// a.NotSubset([1, 3, 4], [1, 2]) +// a.NotSubset({"x": 1, "y": 2}, {"z": 3}) +// a.NotSubset([1, 3, 4], {1: "one", 2: "two"}) +// a.NotSubset({"x": 1, "y": 2}, ["z"]) +func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotSubset(a.t, list, subset, msgAndArgs...) +} + +// NotSubsetf asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// a.NotSubsetf([1, 3, 4], [1, 2], "error message %s", "formatted") +// a.NotSubsetf({"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") +// a.NotSubsetf([1, 3, 4], {1: "one", 2: "two"}, "error message %s", "formatted") +// a.NotSubsetf({"x": 1, "y": 2}, ["z"], "error message %s", "formatted") +func (a *Assertions) NotSubsetf(list interface{}, subset interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotSubsetf(a.t, list, subset, msg, args...) +} + +// NotZero asserts that i is not the zero value for its type. +func (a *Assertions) NotZero(i interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotZero(a.t, i, msgAndArgs...) +} + +// NotZerof asserts that i is not the zero value for its type. +func (a *Assertions) NotZerof(i interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotZerof(a.t, i, msg, args...) +} + +// Panics asserts that the code inside the specified PanicTestFunc panics. +// +// a.Panics(func(){ GoCrazy() }) +func (a *Assertions) Panics(f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Panics(a.t, f, msgAndArgs...) +} + +// PanicsWithError asserts that the code inside the specified PanicTestFunc +// panics, and that the recovered panic value is an error that satisfies the +// EqualError comparison. +// +// a.PanicsWithError("crazy error", func(){ GoCrazy() }) +func (a *Assertions) PanicsWithError(errString string, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + PanicsWithError(a.t, errString, f, msgAndArgs...) +} + +// PanicsWithErrorf asserts that the code inside the specified PanicTestFunc +// panics, and that the recovered panic value is an error that satisfies the +// EqualError comparison. +// +// a.PanicsWithErrorf("crazy error", func(){ GoCrazy() }, "error message %s", "formatted") +func (a *Assertions) PanicsWithErrorf(errString string, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + PanicsWithErrorf(a.t, errString, f, msg, args...) +} + +// PanicsWithValue asserts that the code inside the specified PanicTestFunc panics, and that +// the recovered panic value equals the expected panic value. +// +// a.PanicsWithValue("crazy error", func(){ GoCrazy() }) +func (a *Assertions) PanicsWithValue(expected interface{}, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + PanicsWithValue(a.t, expected, f, msgAndArgs...) +} + +// PanicsWithValuef asserts that the code inside the specified PanicTestFunc panics, and that +// the recovered panic value equals the expected panic value. +// +// a.PanicsWithValuef("crazy error", func(){ GoCrazy() }, "error message %s", "formatted") +func (a *Assertions) PanicsWithValuef(expected interface{}, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + PanicsWithValuef(a.t, expected, f, msg, args...) +} + +// Panicsf asserts that the code inside the specified PanicTestFunc panics. +// +// a.Panicsf(func(){ GoCrazy() }, "error message %s", "formatted") +func (a *Assertions) Panicsf(f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Panicsf(a.t, f, msg, args...) +} + +// Positive asserts that the specified element is positive +// +// a.Positive(1) +// a.Positive(1.23) +func (a *Assertions) Positive(e interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Positive(a.t, e, msgAndArgs...) +} + +// Positivef asserts that the specified element is positive +// +// a.Positivef(1, "error message %s", "formatted") +// a.Positivef(1.23, "error message %s", "formatted") +func (a *Assertions) Positivef(e interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Positivef(a.t, e, msg, args...) +} + +// Regexp asserts that a specified regexp matches a string. +// +// a.Regexp(regexp.MustCompile("start"), "it's starting") +// a.Regexp("start...$", "it's not starting") +func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Regexp(a.t, rx, str, msgAndArgs...) +} + +// Regexpf asserts that a specified regexp matches a string. +// +// a.Regexpf(regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") +// a.Regexpf("start...$", "it's not starting", "error message %s", "formatted") +func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Regexpf(a.t, rx, str, msg, args...) +} + +// Same asserts that two pointers reference the same object. +// +// a.Same(ptr1, ptr2) +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) Same(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Same(a.t, expected, actual, msgAndArgs...) +} + +// Samef asserts that two pointers reference the same object. +// +// a.Samef(ptr1, ptr2, "error message %s", "formatted") +// +// Both arguments must be pointer variables. Pointer variable sameness is +// determined based on the equality of both type and value. +func (a *Assertions) Samef(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Samef(a.t, expected, actual, msg, args...) +} + +// Subset asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// a.Subset([1, 2, 3], [1, 2]) +// a.Subset({"x": 1, "y": 2}, {"x": 1}) +// a.Subset([1, 2, 3], {1: "one", 2: "two"}) +// a.Subset({"x": 1, "y": 2}, ["x"]) +func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Subset(a.t, list, subset, msgAndArgs...) +} + +// Subsetf asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. +// +// a.Subsetf([1, 2, 3], [1, 2], "error message %s", "formatted") +// a.Subsetf({"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") +// a.Subsetf([1, 2, 3], {1: "one", 2: "two"}, "error message %s", "formatted") +// a.Subsetf({"x": 1, "y": 2}, ["x"], "error message %s", "formatted") +func (a *Assertions) Subsetf(list interface{}, subset interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Subsetf(a.t, list, subset, msg, args...) +} + +// True asserts that the specified value is true. +// +// a.True(myBool) +func (a *Assertions) True(value bool, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + True(a.t, value, msgAndArgs...) +} + +// Truef asserts that the specified value is true. +// +// a.Truef(myBool, "error message %s", "formatted") +func (a *Assertions) Truef(value bool, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Truef(a.t, value, msg, args...) +} + +// WithinDuration asserts that the two times are within duration delta of each other. +// +// a.WithinDuration(time.Now(), time.Now(), 10*time.Second) +func (a *Assertions) WithinDuration(expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + WithinDuration(a.t, expected, actual, delta, msgAndArgs...) +} + +// WithinDurationf asserts that the two times are within duration delta of each other. +// +// a.WithinDurationf(time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") +func (a *Assertions) WithinDurationf(expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + WithinDurationf(a.t, expected, actual, delta, msg, args...) +} + +// WithinRange asserts that a time is within a time range (inclusive). +// +// a.WithinRange(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) +func (a *Assertions) WithinRange(actual time.Time, start time.Time, end time.Time, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + WithinRange(a.t, actual, start, end, msgAndArgs...) +} + +// WithinRangef asserts that a time is within a time range (inclusive). +// +// a.WithinRangef(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") +func (a *Assertions) WithinRangef(actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + WithinRangef(a.t, actual, start, end, msg, args...) +} + +// YAMLEq asserts that two YAML strings are equivalent. +func (a *Assertions) YAMLEq(expected string, actual string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + YAMLEq(a.t, expected, actual, msgAndArgs...) +} + +// YAMLEqf asserts that two YAML strings are equivalent. +func (a *Assertions) YAMLEqf(expected string, actual string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + YAMLEqf(a.t, expected, actual, msg, args...) +} + +// Zero asserts that i is the zero value for its type. +func (a *Assertions) Zero(i interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Zero(a.t, i, msgAndArgs...) +} + +// Zerof asserts that i is the zero value for its type. +func (a *Assertions) Zerof(i interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + Zerof(a.t, i, msg, args...) +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/require/require_forward.go.tmpl b/scraper-go/vendor/github.com/stretchr/testify/require/require_forward.go.tmpl new file mode 100644 index 00000000..54124df1 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/require/require_forward.go.tmpl @@ -0,0 +1,5 @@ +{{.CommentWithoutT "a"}} +func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) { + if h, ok := a.t.(tHelper); ok { h.Helper() } + {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) +} diff --git a/scraper-go/vendor/github.com/stretchr/testify/require/requirements.go b/scraper-go/vendor/github.com/stretchr/testify/require/requirements.go new file mode 100644 index 00000000..6b7ce929 --- /dev/null +++ b/scraper-go/vendor/github.com/stretchr/testify/require/requirements.go @@ -0,0 +1,29 @@ +package require + +// TestingT is an interface wrapper around *testing.T +type TestingT interface { + Errorf(format string, args ...interface{}) + FailNow() +} + +type tHelper = interface { + Helper() +} + +// ComparisonAssertionFunc is a common function prototype when comparing two values. Can be useful +// for table driven tests. +type ComparisonAssertionFunc func(TestingT, interface{}, interface{}, ...interface{}) + +// ValueAssertionFunc is a common function prototype when validating a single value. Can be useful +// for table driven tests. +type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) + +// BoolAssertionFunc is a common function prototype when validating a bool value. Can be useful +// for table driven tests. +type BoolAssertionFunc func(TestingT, bool, ...interface{}) + +// ErrorAssertionFunc is a common function prototype when validating an error value. Can be useful +// for table driven tests. +type ErrorAssertionFunc func(TestingT, error, ...interface{}) + +//go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=require -template=require.go.tmpl -include-format-funcs" diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/.gitignore b/scraper-go/vendor/github.com/yuin/gopher-lua/.gitignore new file mode 100644 index 00000000..485dee64 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/LICENSE b/scraper-go/vendor/github.com/yuin/gopher-lua/LICENSE new file mode 100644 index 00000000..4daf480a --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Yusuke Inuzuka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/Makefile b/scraper-go/vendor/github.com/yuin/gopher-lua/Makefile new file mode 100644 index 00000000..6d9e55c3 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/Makefile @@ -0,0 +1,10 @@ +.PHONY: build test glua + +build: + ./_tools/go-inline *.go && go fmt . && go build + +glua: *.go pm/*.go cmd/glua/glua.go + ./_tools/go-inline *.go && go fmt . && go build cmd/glua/glua.go + +test: + ./_tools/go-inline *.go && go fmt . && go test diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/README.rst b/scraper-go/vendor/github.com/yuin/gopher-lua/README.rst new file mode 100644 index 00000000..cc936311 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/README.rst @@ -0,0 +1,890 @@ + +=============================================================================== +GopherLua: VM and compiler for Lua in Go. +=============================================================================== + +.. image:: https://pkg.go.dev/badge/github.com/yuin/gopher-lua.svg + :target: https://pkg.go.dev/github.com/yuin/gopher-lua + +.. image:: https://github.com/yuin/gopher-lua/workflows/test/badge.svg?branch=master&event=push + :target: https://github.com/yuin/gopher-lua/actions?query=workflow:test + +.. image:: https://coveralls.io/repos/github/yuin/gopher-lua/badge.svg?branch=master + :target: https://coveralls.io/github/yuin/gopher-lua + +.. image:: https://badges.gitter.im/Join%20Chat.svg + :alt: Join the chat at https://gitter.im/yuin/gopher-lua + :target: https://gitter.im/yuin/gopher-lua?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge + +| + + +GopherLua is a Lua5.1(+ `goto` statement in Lua5.2) VM and compiler written in Go. GopherLua has a same goal +with Lua: **Be a scripting language with extensible semantics** . It provides +Go APIs that allow you to easily embed a scripting language to your Go host +programs. + +.. contents:: + :depth: 1 + +---------------------------------------------------------------- +Design principle +---------------------------------------------------------------- + +- Be a scripting language with extensible semantics. +- User-friendly Go API + - The stack based API like the one used in the original Lua + implementation will cause a performance improvements in GopherLua + (It will reduce memory allocations and concrete type <-> interface conversions). + GopherLua API is **not** the stack based API. + GopherLua give preference to the user-friendliness over the performance. + +---------------------------------------------------------------- +How about performance? +---------------------------------------------------------------- +GopherLua is not fast but not too slow, I think. + +GopherLua has almost equivalent ( or little bit better ) performance as Python3 on micro benchmarks. + +There are some benchmarks on the `wiki page `_ . + +---------------------------------------------------------------- +Installation +---------------------------------------------------------------- + +.. code-block:: bash + + go get github.com/yuin/gopher-lua + +GopherLua supports >= Go1.9. + +---------------------------------------------------------------- +Usage +---------------------------------------------------------------- +GopherLua APIs perform in much the same way as Lua, **but the stack is used only +for passing arguments and receiving returned values.** + +GopherLua supports channel operations. See **"Goroutines"** section. + +Import a package. + +.. code-block:: go + + import ( + "github.com/yuin/gopher-lua" + ) + +Run scripts in the VM. + +.. code-block:: go + + L := lua.NewState() + defer L.Close() + if err := L.DoString(`print("hello")`); err != nil { + panic(err) + } + +.. code-block:: go + + L := lua.NewState() + defer L.Close() + if err := L.DoFile("hello.lua"); err != nil { + panic(err) + } + +Refer to `Lua Reference Manual `_ and `Go doc `_ for further information. + +Note that elements that are not commented in `Go doc `_ equivalent to `Lua Reference Manual `_ , except GopherLua uses objects instead of Lua stack indices. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Data model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +All data in a GopherLua program is an ``LValue`` . ``LValue`` is an interface +type that has following methods. + +- ``String() string`` +- ``Type() LValueType`` + + +Objects implement an LValue interface are + +================ ========================= ================== ======================= + Type name Go type Type() value Constants +================ ========================= ================== ======================= + ``LNilType`` (constants) ``LTNil`` ``LNil`` + ``LBool`` (constants) ``LTBool`` ``LTrue``, ``LFalse`` + ``LNumber`` float64 ``LTNumber`` ``-`` + ``LString`` string ``LTString`` ``-`` + ``LFunction`` struct pointer ``LTFunction`` ``-`` + ``LUserData`` struct pointer ``LTUserData`` ``-`` + ``LState`` struct pointer ``LTThread`` ``-`` + ``LTable`` struct pointer ``LTTable`` ``-`` + ``LChannel`` chan LValue ``LTChannel`` ``-`` +================ ========================= ================== ======================= + +You can test an object type in Go way(type assertion) or using a ``Type()`` value. + +.. code-block:: go + + lv := L.Get(-1) // get the value at the top of the stack + if str, ok := lv.(lua.LString); ok { + // lv is LString + fmt.Println(string(str)) + } + if lv.Type() != lua.LTString { + panic("string required.") + } + +.. code-block:: go + + lv := L.Get(-1) // get the value at the top of the stack + if tbl, ok := lv.(*lua.LTable); ok { + // lv is LTable + fmt.Println(L.ObjLen(tbl)) + } + +Note that ``LBool`` , ``LNumber`` , ``LString`` is not a pointer. + +To test ``LNilType`` and ``LBool``, You **must** use pre-defined constants. + +.. code-block:: go + + lv := L.Get(-1) // get the value at the top of the stack + + if lv == lua.LTrue { // correct + } + + if bl, ok := lv.(lua.LBool); ok && bool(bl) { // wrong + } + +In Lua, both ``nil`` and ``false`` make a condition false. ``LVIsFalse`` and ``LVAsBool`` implement this specification. + +.. code-block:: go + + lv := L.Get(-1) // get the value at the top of the stack + if lua.LVIsFalse(lv) { // lv is nil or false + } + + if lua.LVAsBool(lv) { // lv is neither nil nor false + } + +Objects that based on go structs(``LFunction``. ``LUserData``, ``LTable``) +have some public methods and fields. You can use these methods and fields for +performance and debugging, but there are some limitations. + +- Metatable does not work. +- No error handlings. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Callstack & Registry size +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The size of an ``LState``'s callstack controls the maximum call depth for Lua functions within a script (Go function calls do not count). + +The registry of an ``LState`` implements stack storage for calling functions (both Lua and Go functions) and also for temporary variables in expressions. Its storage requirements will increase with callstack usage and also with code complexity. + +Both the registry and the callstack can be set to either a fixed size or to auto size. + +When you have a large number of ``LStates`` instantiated in a process, it's worth taking the time to tune the registry and callstack options. + ++++++++++ +Registry ++++++++++ + +The registry can have an initial size, a maximum size and a step size configured on a per ``LState`` basis. This will allow the registry to grow as needed. It will not shrink again after growing. + +.. code-block:: go + + L := lua.NewState(lua.Options{ + RegistrySize: 1024 * 20, // this is the initial size of the registry + RegistryMaxSize: 1024 * 80, // this is the maximum size that the registry can grow to. If set to `0` (the default) then the registry will not auto grow + RegistryGrowStep: 32, // this is how much to step up the registry by each time it runs out of space. The default is `32`. + }) + defer L.Close() + +A registry which is too small for a given script will ultimately result in a panic. A registry which is too big will waste memory (which can be significant if many ``LStates`` are instantiated). +Auto growing registries incur a small performance hit at the point they are resized but will not otherwise affect performance. + ++++++++++ +Callstack ++++++++++ + +The callstack can operate in two different modes, fixed or auto size. +A fixed size callstack has the highest performance and has a fixed memory overhead. +An auto sizing callstack will allocate and release callstack pages on demand which will ensure the minimum amount of memory is in use at any time. The downside is it will incur a small performance impact every time a new page of callframes is allocated. +By default an ``LState`` will allocate and free callstack frames in pages of 8, so the allocation overhead is not incurred on every function call. It is very likely that the performance impact of an auto resizing callstack will be negligible for most use cases. + +.. code-block:: go + + L := lua.NewState(lua.Options{ + CallStackSize: 120, // this is the maximum callstack size of this LState + MinimizeStackMemory: true, // Defaults to `false` if not specified. If set, the callstack will auto grow and shrink as needed up to a max of `CallStackSize`. If not set, the callstack will be fixed at `CallStackSize`. + }) + defer L.Close() + +++++++++++++++++ +Option defaults +++++++++++++++++ + +The above examples show how to customize the callstack and registry size on a per ``LState`` basis. You can also adjust some defaults for when options are not specified by altering the values of ``lua.RegistrySize``, ``lua.RegistryGrowStep`` and ``lua.CallStackSize``. + +An ``LState`` object that has been created by ``*LState#NewThread()`` inherits the callstack & registry size from the parent ``LState`` object. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Miscellaneous lua.NewState options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- **Options.SkipOpenLibs bool(default false)** + - By default, GopherLua opens all built-in libraries when new LState is created. + - You can skip this behaviour by setting this to ``true`` . + - Using the various `OpenXXX(L *LState) int` functions you can open only those libraries that you require, for an example see below. +- **Options.IncludeGoStackTrace bool(default false)** + - By default, GopherLua does not show Go stack traces when panics occur. + - You can get Go stack traces by setting this to ``true`` . + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Refer to `Lua Reference Manual `_ and `Go doc(LState methods) `_ for further information. + ++++++++++++++++++++++++++++++++++++++++++ +Calling Go from Lua ++++++++++++++++++++++++++++++++++++++++++ + +.. code-block:: go + + func Double(L *lua.LState) int { + lv := L.ToInt(1) /* get argument */ + L.Push(lua.LNumber(lv * 2)) /* push result */ + return 1 /* number of results */ + } + + func main() { + L := lua.NewState() + defer L.Close() + L.SetGlobal("double", L.NewFunction(Double)) /* Original lua_setglobal uses stack... */ + } + +.. code-block:: lua + + print(double(20)) -- > "40" + +Any function registered with GopherLua is a ``lua.LGFunction``, defined in ``value.go`` + +.. code-block:: go + + type LGFunction func(*LState) int + +Working with coroutines. + +.. code-block:: go + + co, _ := L.NewThread() /* create a new thread */ + fn := L.GetGlobal("coro").(*lua.LFunction) /* get function from lua */ + for { + st, err, values := L.Resume(co, fn) + if st == lua.ResumeError { + fmt.Println("yield break(error)") + fmt.Println(err.Error()) + break + } + + for i, lv := range values { + fmt.Printf("%v : %v\n", i, lv) + } + + if st == lua.ResumeOK { + fmt.Println("yield break(ok)") + break + } + } + ++++++++++++++++++++++++++++++++++++++++++ +Opening a subset of builtin modules ++++++++++++++++++++++++++++++++++++++++++ + +The following demonstrates how to open a subset of the built-in modules in Lua, say for example to avoid enabling modules with access to local files or system calls. + +main.go + +.. code-block:: go + + func main() { + L := lua.NewState(lua.Options{SkipOpenLibs: true}) + defer L.Close() + for _, pair := range []struct { + n string + f lua.LGFunction + }{ + {lua.LoadLibName, lua.OpenPackage}, // Must be first + {lua.BaseLibName, lua.OpenBase}, + {lua.TabLibName, lua.OpenTable}, + } { + if err := L.CallByParam(lua.P{ + Fn: L.NewFunction(pair.f), + NRet: 0, + Protect: true, + }, lua.LString(pair.n)); err != nil { + panic(err) + } + } + if err := L.DoFile("main.lua"); err != nil { + panic(err) + } + } + ++++++++++++++++++++++++++++++++++++++++++ +Creating a module by Go ++++++++++++++++++++++++++++++++++++++++++ + +mymodule.go + +.. code-block:: go + + package mymodule + + import ( + "github.com/yuin/gopher-lua" + ) + + func Loader(L *lua.LState) int { + // register functions to the table + mod := L.SetFuncs(L.NewTable(), exports) + // register other stuff + L.SetField(mod, "name", lua.LString("value")) + + // returns the module + L.Push(mod) + return 1 + } + + var exports = map[string]lua.LGFunction{ + "myfunc": myfunc, + } + + func myfunc(L *lua.LState) int { + return 0 + } + +mymain.go + +.. code-block:: go + + package main + + import ( + "./mymodule" + "github.com/yuin/gopher-lua" + ) + + func main() { + L := lua.NewState() + defer L.Close() + L.PreloadModule("mymodule", mymodule.Loader) + if err := L.DoFile("main.lua"); err != nil { + panic(err) + } + } + +main.lua + +.. code-block:: lua + + local m = require("mymodule") + m.myfunc() + print(m.name) + + ++++++++++++++++++++++++++++++++++++++++++ +Calling Lua from Go ++++++++++++++++++++++++++++++++++++++++++ + +.. code-block:: go + + L := lua.NewState() + defer L.Close() + if err := L.DoFile("double.lua"); err != nil { + panic(err) + } + if err := L.CallByParam(lua.P{ + Fn: L.GetGlobal("double"), + NRet: 1, + Protect: true, + }, lua.LNumber(10)); err != nil { + panic(err) + } + ret := L.Get(-1) // returned value + L.Pop(1) // remove received value + +If ``Protect`` is false, GopherLua will panic instead of returning an ``error`` value. + ++++++++++++++++++++++++++++++++++++++++++ +User-Defined types ++++++++++++++++++++++++++++++++++++++++++ +You can extend GopherLua with new types written in Go. +``LUserData`` is provided for this purpose. + +.. code-block:: go + + type Person struct { + Name string + } + + const luaPersonTypeName = "person" + + // Registers my person type to given L. + func registerPersonType(L *lua.LState) { + mt := L.NewTypeMetatable(luaPersonTypeName) + L.SetGlobal("person", mt) + // static attributes + L.SetField(mt, "new", L.NewFunction(newPerson)) + // methods + L.SetField(mt, "__index", L.SetFuncs(L.NewTable(), personMethods)) + } + + // Constructor + func newPerson(L *lua.LState) int { + person := &Person{L.CheckString(1)} + ud := L.NewUserData() + ud.Value = person + L.SetMetatable(ud, L.GetTypeMetatable(luaPersonTypeName)) + L.Push(ud) + return 1 + } + + // Checks whether the first lua argument is a *LUserData with *Person and returns this *Person. + func checkPerson(L *lua.LState) *Person { + ud := L.CheckUserData(1) + if v, ok := ud.Value.(*Person); ok { + return v + } + L.ArgError(1, "person expected") + return nil + } + + var personMethods = map[string]lua.LGFunction{ + "name": personGetSetName, + } + + // Getter and setter for the Person#Name + func personGetSetName(L *lua.LState) int { + p := checkPerson(L) + if L.GetTop() == 2 { + p.Name = L.CheckString(2) + return 0 + } + L.Push(lua.LString(p.Name)) + return 1 + } + + func main() { + L := lua.NewState() + defer L.Close() + registerPersonType(L) + if err := L.DoString(` + p = person.new("Steeve") + print(p:name()) -- "Steeve" + p:name("Alice") + print(p:name()) -- "Alice" + `); err != nil { + panic(err) + } + } + ++++++++++++++++++++++++++++++++++++++++++ +Terminating a running LState ++++++++++++++++++++++++++++++++++++++++++ +GopherLua supports the `Go Concurrency Patterns: Context `_ . + + +.. code-block:: go + + L := lua.NewState() + defer L.Close() + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + // set the context to our LState + L.SetContext(ctx) + err := L.DoString(` + local clock = os.clock + function sleep(n) -- seconds + local t0 = clock() + while clock() - t0 <= n do end + end + sleep(3) + `) + // err.Error() contains "context deadline exceeded" + +With coroutines + +.. code-block:: go + + L := lua.NewState() + defer L.Close() + ctx, cancel := context.WithCancel(context.Background()) + L.SetContext(ctx) + defer cancel() + L.DoString(` + function coro() + local i = 0 + while true do + coroutine.yield(i) + i = i+1 + end + return i + end + `) + co, cocancel := L.NewThread() + defer cocancel() + fn := L.GetGlobal("coro").(*LFunction) + + _, err, values := L.Resume(co, fn) // err is nil + + cancel() // cancel the parent context + + _, err, values = L.Resume(co, fn) // err is NOT nil : child context was canceled + +**Note that using a context causes performance degradation.** + +.. code-block:: + + time ./glua-with-context.exe fib.lua + 9227465 + 0.01s user 0.11s system 1% cpu 7.505 total + + time ./glua-without-context.exe fib.lua + 9227465 + 0.01s user 0.01s system 0% cpu 5.306 total + ++++++++++++++++++++++++++++++++++++++++++ +Sharing Lua byte code between LStates ++++++++++++++++++++++++++++++++++++++++++ +Calling ``DoFile`` will load a Lua script, compile it to byte code and run the byte code in a ``LState``. + +If you have multiple ``LStates`` which are all required to run the same script, you can share the byte code between them, +which will save on memory. +Sharing byte code is safe as it is read only and cannot be altered by lua scripts. + +.. code-block:: go + + // CompileLua reads the passed lua file from disk and compiles it. + func CompileLua(filePath string) (*lua.FunctionProto, error) { + file, err := os.Open(filePath) + defer file.Close() + if err != nil { + return nil, err + } + reader := bufio.NewReader(file) + chunk, err := parse.Parse(reader, filePath) + if err != nil { + return nil, err + } + proto, err := lua.Compile(chunk, filePath) + if err != nil { + return nil, err + } + return proto, nil + } + + // DoCompiledFile takes a FunctionProto, as returned by CompileLua, and runs it in the LState. It is equivalent + // to calling DoFile on the LState with the original source file. + func DoCompiledFile(L *lua.LState, proto *lua.FunctionProto) error { + lfunc := L.NewFunctionFromProto(proto) + L.Push(lfunc) + return L.PCall(0, lua.MultRet, nil) + } + + // Example shows how to share the compiled byte code from a lua script between multiple VMs. + func Example() { + codeToShare := CompileLua("mylua.lua") + a := lua.NewState() + b := lua.NewState() + c := lua.NewState() + DoCompiledFile(a, codeToShare) + DoCompiledFile(b, codeToShare) + DoCompiledFile(c, codeToShare) + } + ++++++++++++++++++++++++++++++++++++++++++ +Goroutines ++++++++++++++++++++++++++++++++++++++++++ +The ``LState`` is not goroutine-safe. It is recommended to use one LState per goroutine and communicate between goroutines by using channels. + +Channels are represented by ``channel`` objects in GopherLua. And a ``channel`` table provides functions for performing channel operations. + +Some objects can not be sent over channels due to having non-goroutine-safe objects inside itself. + +- a thread(state) +- a function +- an userdata +- a table with a metatable + +You **must not** send these objects from Go APIs to channels. + + + +.. code-block:: go + + func receiver(ch, quit chan lua.LValue) { + L := lua.NewState() + defer L.Close() + L.SetGlobal("ch", lua.LChannel(ch)) + L.SetGlobal("quit", lua.LChannel(quit)) + if err := L.DoString(` + local exit = false + while not exit do + channel.select( + {"|<-", ch, function(ok, v) + if not ok then + print("channel closed") + exit = true + else + print("received:", v) + end + end}, + {"|<-", quit, function(ok, v) + print("quit") + exit = true + end} + ) + end + `); err != nil { + panic(err) + } + } + + func sender(ch, quit chan lua.LValue) { + L := lua.NewState() + defer L.Close() + L.SetGlobal("ch", lua.LChannel(ch)) + L.SetGlobal("quit", lua.LChannel(quit)) + if err := L.DoString(` + ch:send("1") + ch:send("2") + `); err != nil { + panic(err) + } + ch <- lua.LString("3") + quit <- lua.LTrue + } + + func main() { + ch := make(chan lua.LValue) + quit := make(chan lua.LValue) + go receiver(ch, quit) + go sender(ch, quit) + time.Sleep(3 * time.Second) + } + +''''''''''''''' +Go API +''''''''''''''' + +``ToChannel``, ``CheckChannel``, ``OptChannel`` are available. + +Refer to `Go doc(LState methods) `_ for further information. + +''''''''''''''' +Lua API +''''''''''''''' + +- **channel.make([buf:int]) -> ch:channel** + - Create new channel that has a buffer size of ``buf``. By default, ``buf`` is 0. + +- **channel.select(case:table [, case:table, case:table ...]) -> {index:int, recv:any, ok}** + - Same as the ``select`` statement in Go. It returns the index of the chosen case and, if that + case was a receive operation, the value received and a boolean indicating whether the channel has been closed. + - ``case`` is a table that outlined below. + - receiving: `{"|<-", ch:channel [, handler:func(ok, data:any)]}` + - sending: `{"<-|", ch:channel, data:any [, handler:func(data:any)]}` + - default: `{"default" [, handler:func()]}` + +``channel.select`` examples: + +.. code-block:: lua + + local idx, recv, ok = channel.select( + {"|<-", ch1}, + {"|<-", ch2} + ) + if not ok then + print("closed") + elseif idx == 1 then -- received from ch1 + print(recv) + elseif idx == 2 then -- received from ch2 + print(recv) + end + +.. code-block:: lua + + channel.select( + {"|<-", ch1, function(ok, data) + print(ok, data) + end}, + {"<-|", ch2, "value", function(data) + print(data) + end}, + {"default", function() + print("default action") + end} + ) + +- **channel:send(data:any)** + - Send ``data`` over the channel. +- **channel:receive() -> ok:bool, data:any** + - Receive some data over the channel. +- **channel:close()** + - Close the channel. + +'''''''''''''''''''''''''''''' +The LState pool pattern +'''''''''''''''''''''''''''''' +To create per-thread LState instances, You can use the ``sync.Pool`` like mechanism. + +.. code-block:: go + + type lStatePool struct { + m sync.Mutex + saved []*lua.LState + } + + func (pl *lStatePool) Get() *lua.LState { + pl.m.Lock() + defer pl.m.Unlock() + n := len(pl.saved) + if n == 0 { + return pl.New() + } + x := pl.saved[n-1] + pl.saved = pl.saved[0 : n-1] + return x + } + + func (pl *lStatePool) New() *lua.LState { + L := lua.NewState() + // setting the L up here. + // load scripts, set global variables, share channels, etc... + return L + } + + func (pl *lStatePool) Put(L *lua.LState) { + pl.m.Lock() + defer pl.m.Unlock() + pl.saved = append(pl.saved, L) + } + + func (pl *lStatePool) Shutdown() { + for _, L := range pl.saved { + L.Close() + } + } + + // Global LState pool + var luaPool = &lStatePool{ + saved: make([]*lua.LState, 0, 4), + } + +Now, you can get per-thread LState objects from the ``luaPool`` . + +.. code-block:: go + + func MyWorker() { + L := luaPool.Get() + defer luaPool.Put(L) + /* your code here */ + } + + func main() { + defer luaPool.Shutdown() + go MyWorker() + go MyWorker() + /* etc... */ + } + + +---------------------------------------------------------------- +Differences between Lua and GopherLua +---------------------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Goroutines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- GopherLua supports channel operations. + - GopherLua has a type named ``channel``. + - The ``channel`` table provides functions for performing channel operations. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Unsupported functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``string.dump`` +- ``os.setlocale`` +- ``lua_Debug.namewhat`` +- ``package.loadlib`` +- debug hooks + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Miscellaneous notes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``collectgarbage`` does not take any arguments and runs the garbage collector for the entire Go program. +- ``file:setvbuf`` does not support a line buffering. +- Daylight saving time is not supported. +- GopherLua has a function to set an environment variable : ``os.setenv(name, value)`` +- GopherLua support ``goto`` and ``::label::`` statement in Lua5.2. + - `goto` is a keyword and not a valid variable name. + +---------------------------------------------------------------- +Standalone interpreter +---------------------------------------------------------------- +Lua has an interpreter called ``lua`` . GopherLua has an interpreter called ``glua`` . + +.. code-block:: bash + + go get github.com/yuin/gopher-lua/cmd/glua + +``glua`` has same options as ``lua`` . + +---------------------------------------------------------------- +How to Contribute +---------------------------------------------------------------- +See `Guidlines for contributors `_ . + +---------------------------------------------------------------- +Libraries for GopherLua +---------------------------------------------------------------- + +- `gopher-luar `_ : Simplifies data passing to and from gopher-lua +- `gluamapper `_ : Mapping a Lua table to a Go struct +- `gluare `_ : Regular expressions for gopher-lua +- `gluahttp `_ : HTTP request module for gopher-lua +- `gopher-json `_ : A simple JSON encoder/decoder for gopher-lua +- `gluayaml `_ : Yaml parser for gopher-lua +- `glua-lfs `_ : Partially implements the luafilesystem module for gopher-lua +- `gluaurl `_ : A url parser/builder module for gopher-lua +- `gluahttpscrape `_ : A simple HTML scraper module for gopher-lua +- `gluaxmlpath `_ : An xmlpath module for gopher-lua +- `gmoonscript `_ : Moonscript Compiler for the Gopher Lua VM +- `loguago `_ : Zerolog wrapper for Gopher-Lua +- `gluacrypto `_ : A native Go implementation of crypto library for the GopherLua VM. +- `gluasql `_ : A native Go implementation of SQL client for the GopherLua VM. +- `purr `_ : A http mock testing tool. +- `vadv/gopher-lua-libs `_ : Some usefull libraries for GopherLua VM. +- `gluaperiphery `_ : A periphery library for the GopherLua VM (GPIO, SPI, I2C, MMIO, and Serial peripheral I/O for Linux). +- `glua-async `_ : An async/await implement for gopher-lua. +- `gopherlua-debugger `_ : A debugger for gopher-lua +- `gluamahonia `_ : An encoding converter for gopher-lua +---------------------------------------------------------------- +Donation +---------------------------------------------------------------- + +BTC: 1NEDSyUmo4SMTDP83JJQSWi1MvQUGGNMZB + +---------------------------------------------------------------- +License +---------------------------------------------------------------- +MIT + +---------------------------------------------------------------- +Author +---------------------------------------------------------------- +Yusuke Inuzuka diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/_state.go b/scraper-go/vendor/github.com/yuin/gopher-lua/_state.go new file mode 100644 index 00000000..6e9febfb --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/_state.go @@ -0,0 +1,2093 @@ +package lua + +import ( + "context" + "fmt" + "io" + "math" + "os" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/yuin/gopher-lua/parse" +) + +const MultRet = -1 +const RegistryIndex = -10000 +const EnvironIndex = -10001 +const GlobalsIndex = -10002 + +/* ApiError {{{ */ + +type ApiError struct { + Type ApiErrorType + Object LValue + StackTrace string + // Underlying error. This attribute is set only if the Type is ApiErrorFile or ApiErrorSyntax + Cause error +} + +func newApiError(code ApiErrorType, object LValue) *ApiError { + return &ApiError{code, object, "", nil} +} + +func newApiErrorS(code ApiErrorType, message string) *ApiError { + return newApiError(code, LString(message)) +} + +func newApiErrorE(code ApiErrorType, err error) *ApiError { + return &ApiError{code, LString(err.Error()), "", err} +} + +func (e *ApiError) Error() string { + if len(e.StackTrace) > 0 { + return fmt.Sprintf("%s\n%s", e.Object.String(), e.StackTrace) + } + return e.Object.String() +} + +type ApiErrorType int + +const ( + ApiErrorSyntax ApiErrorType = iota + ApiErrorFile + ApiErrorRun + ApiErrorError + ApiErrorPanic +) + +/* }}} */ + +/* ResumeState {{{ */ + +type ResumeState int + +const ( + ResumeOK ResumeState = iota + ResumeYield + ResumeError +) + +/* }}} */ + +/* P {{{ */ + +type P struct { + Fn LValue + NRet int + Protect bool + Handler *LFunction +} + +/* }}} */ + +/* Options {{{ */ + +// Options is a configuration that is used to create a new LState. +type Options struct { + // Call stack size. This defaults to `lua.CallStackSize`. + CallStackSize int + // Data stack size. This defaults to `lua.RegistrySize`. + RegistrySize int + // Allow the registry to grow from the registry size specified up to a value of RegistryMaxSize. A value of 0 + // indicates no growth is permitted. The registry will not shrink again after any growth. + RegistryMaxSize int + // If growth is enabled, step up by an additional `RegistryGrowStep` each time to avoid having to resize too often. + // This defaults to `lua.RegistryGrowStep` + RegistryGrowStep int + // Controls whether or not libraries are opened by default + SkipOpenLibs bool + // Tells whether a Go stacktrace should be included in a Lua stacktrace when panics occur. + IncludeGoStackTrace bool + // If `MinimizeStackMemory` is set, the call stack will be automatically grown or shrank up to a limit of + // `CallStackSize` in order to minimize memory usage. This does incur a slight performance penalty. + MinimizeStackMemory bool +} + +/* }}} */ + +/* Debug {{{ */ + +type Debug struct { + frame *callFrame + Name string + What string + Source string + CurrentLine int + NUpvalues int + LineDefined int + LastLineDefined int +} + +/* }}} */ + +/* callFrame {{{ */ + +type callFrame struct { + Idx int + Fn *LFunction + Parent *callFrame + Pc int + Base int + LocalBase int + ReturnBase int + NArgs int + NRet int + TailCall int +} + +type callFrameStack interface { + Push(v callFrame) + Pop() *callFrame + Last() *callFrame + + SetSp(sp int) + Sp() int + At(sp int) *callFrame + + IsFull() bool + IsEmpty() bool + + FreeAll() +} + +type fixedCallFrameStack struct { + array []callFrame + sp int +} + +func newFixedCallFrameStack(size int) callFrameStack { + return &fixedCallFrameStack{ + array: make([]callFrame, size), + sp: 0, + } +} + +func (cs *fixedCallFrameStack) IsEmpty() bool { return cs.sp == 0 } + +func (cs *fixedCallFrameStack) IsFull() bool { return cs.sp == len(cs.array) } + +func (cs *fixedCallFrameStack) Clear() { + cs.sp = 0 +} + +func (cs *fixedCallFrameStack) Push(v callFrame) { + cs.array[cs.sp] = v + cs.array[cs.sp].Idx = cs.sp + cs.sp++ +} + +func (cs *fixedCallFrameStack) Sp() int { + return cs.sp +} + +func (cs *fixedCallFrameStack) SetSp(sp int) { + cs.sp = sp +} + +func (cs *fixedCallFrameStack) Last() *callFrame { + if cs.sp == 0 { + return nil + } + return &cs.array[cs.sp-1] +} + +func (cs *fixedCallFrameStack) At(sp int) *callFrame { + return &cs.array[sp] +} + +func (cs *fixedCallFrameStack) Pop() *callFrame { + cs.sp-- + return &cs.array[cs.sp] +} + +func (cs *fixedCallFrameStack) FreeAll() { + // nothing to do for fixed callframestack +} + +// FramesPerSegment should be a power of 2 constant for performance reasons. It will allow the go compiler to change +// the divs and mods into bitshifts. Max is 256 due to current use of uint8 to count how many frames in a segment are +// used. +const FramesPerSegment = 8 + +type callFrameStackSegment struct { + array [FramesPerSegment]callFrame +} +type segIdx uint16 +type autoGrowingCallFrameStack struct { + segments []*callFrameStackSegment + segIdx segIdx + // segSp is the number of frames in the current segment which are used. Full 'sp' value is segIdx * FramesPerSegment + segSp. + // It points to the next stack slot to use, so 0 means to use the 0th element in the segment, and a value of + // FramesPerSegment indicates that the segment is full and cannot accommodate another frame. + segSp uint8 +} + +var segmentPool sync.Pool + +func newCallFrameStackSegment() *callFrameStackSegment { + seg := segmentPool.Get() + if seg == nil { + return &callFrameStackSegment{} + } + return seg.(*callFrameStackSegment) +} + +func freeCallFrameStackSegment(seg *callFrameStackSegment) { + segmentPool.Put(seg) +} + +// newCallFrameStack allocates a new stack for a lua state, which will auto grow up to a max size of at least maxSize. +// it will actually grow up to the next segment size multiple after maxSize, where the segment size is dictated by +// FramesPerSegment. +func newAutoGrowingCallFrameStack(maxSize int) callFrameStack { + cs := &autoGrowingCallFrameStack{ + segments: make([]*callFrameStackSegment, (maxSize+(FramesPerSegment-1))/FramesPerSegment), + segIdx: 0, + } + cs.segments[0] = newCallFrameStackSegment() + return cs +} + +func (cs *autoGrowingCallFrameStack) IsEmpty() bool { + return cs.segIdx == 0 && cs.segSp == 0 +} + +// IsFull returns true if the stack cannot receive any more stack pushes without overflowing +func (cs *autoGrowingCallFrameStack) IsFull() bool { + return int(cs.segIdx) == len(cs.segments) && cs.segSp >= FramesPerSegment +} + +func (cs *autoGrowingCallFrameStack) Clear() { + for i := segIdx(1); i <= cs.segIdx; i++ { + freeCallFrameStackSegment(cs.segments[i]) + cs.segments[i] = nil + } + cs.segIdx = 0 + cs.segSp = 0 +} + +func (cs *autoGrowingCallFrameStack) FreeAll() { + for i := segIdx(0); i <= cs.segIdx; i++ { + freeCallFrameStackSegment(cs.segments[i]) + cs.segments[i] = nil + } +} + +// Push pushes the passed callFrame onto the stack. it panics if the stack is full, caller should call IsFull() before +// invoking this to avoid this. +func (cs *autoGrowingCallFrameStack) Push(v callFrame) { + curSeg := cs.segments[cs.segIdx] + if cs.segSp >= FramesPerSegment { + // segment full, push new segment if allowed + if cs.segIdx < segIdx(len(cs.segments)-1) { + curSeg = newCallFrameStackSegment() + cs.segIdx++ + cs.segments[cs.segIdx] = curSeg + cs.segSp = 0 + } else { + panic("lua callstack overflow") + } + } + curSeg.array[cs.segSp] = v + curSeg.array[cs.segSp].Idx = int(cs.segSp) + FramesPerSegment*int(cs.segIdx) + cs.segSp++ +} + +// Sp retrieves the current stack depth, which is the number of frames currently pushed on the stack. +func (cs *autoGrowingCallFrameStack) Sp() int { + return int(cs.segSp) + int(cs.segIdx)*FramesPerSegment +} + +// SetSp can be used to rapidly unwind the stack, freeing all stack frames on the way. It should not be used to +// allocate new stack space, use Push() for that. +func (cs *autoGrowingCallFrameStack) SetSp(sp int) { + desiredSegIdx := segIdx(sp / FramesPerSegment) + desiredFramesInLastSeg := uint8(sp % FramesPerSegment) + for { + if cs.segIdx <= desiredSegIdx { + break + } + freeCallFrameStackSegment(cs.segments[cs.segIdx]) + cs.segments[cs.segIdx] = nil + cs.segIdx-- + } + cs.segSp = desiredFramesInLastSeg +} + +func (cs *autoGrowingCallFrameStack) Last() *callFrame { + curSeg := cs.segments[cs.segIdx] + segSp := cs.segSp + if segSp == 0 { + if cs.segIdx == 0 { + return nil + } + curSeg = cs.segments[cs.segIdx-1] + segSp = FramesPerSegment + } + return &curSeg.array[segSp-1] +} + +func (cs *autoGrowingCallFrameStack) At(sp int) *callFrame { + segIdx := segIdx(sp / FramesPerSegment) + frameIdx := uint8(sp % FramesPerSegment) + return &cs.segments[segIdx].array[frameIdx] +} + +// Pop pops off the most recent stack frame and returns it +func (cs *autoGrowingCallFrameStack) Pop() *callFrame { + curSeg := cs.segments[cs.segIdx] + if cs.segSp == 0 { + if cs.segIdx == 0 { + // stack empty + return nil + } + freeCallFrameStackSegment(curSeg) + cs.segments[cs.segIdx] = nil + cs.segIdx-- + cs.segSp = FramesPerSegment + curSeg = cs.segments[cs.segIdx] + } + cs.segSp-- + return &curSeg.array[cs.segSp] +} + +/* }}} */ + +/* registry {{{ */ + +type registryHandler interface { + registryOverflow() +} +type registry struct { + array []LValue + top int + growBy int + maxSize int + alloc *allocator + handler registryHandler +} + +func newRegistry(handler registryHandler, initialSize int, growBy int, maxSize int, alloc *allocator) *registry { + return ®istry{make([]LValue, initialSize), 0, growBy, maxSize, alloc, handler} +} + +func (rg *registry) checkSize(requiredSize int) { // +inline-start + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } +} // +inline-end + +func (rg *registry) resize(requiredSize int) { // +inline-start + newSize := requiredSize + rg.growBy // give some padding + if newSize > rg.maxSize { + newSize = rg.maxSize + } + if newSize < requiredSize { + rg.handler.registryOverflow() + return + } + rg.forceResize(newSize) +} // +inline-end + +func (rg *registry) forceResize(newSize int) { + newSlice := make([]LValue, newSize) + copy(newSlice, rg.array[:rg.top]) // should we copy the area beyond top? there shouldn't be any valid values there so it shouldn't be necessary. + rg.array = newSlice +} + +func (rg *registry) SetTop(topi int) { // +inline-start + // +inline-call rg.checkSize topi + oldtopi := rg.top + rg.top = topi + for i := oldtopi; i < rg.top; i++ { + rg.array[i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + if rg.top < oldtopi { + nilRange := rg.array[rg.top:oldtopi] + for i := range nilRange { + nilRange[i] = nil + } + } + //for i := rg.top; i < oldtop; i++ { + // rg.array[i] = LNil + //} +} // +inline-end + +func (rg *registry) Top() int { + return rg.top +} + +func (rg *registry) Push(v LValue) { + newSize := rg.top + 1 + // +inline-call rg.checkSize newSize + rg.array[rg.top] = v + rg.top++ +} + +func (rg *registry) Pop() LValue { + v := rg.array[rg.top-1] + rg.array[rg.top-1] = LNil + rg.top-- + return v +} + +func (rg *registry) Get(reg int) LValue { + return rg.array[reg] +} + +// CopyRange will move a section of values from index `start` to index `regv` +// It will move `n` values. +// `limit` specifies the maximum end range that can be copied from. If it's set to -1, then it defaults to stopping at +// the top of the registry (values beyond the top are not initialized, so if specifying an alternative `limit` you should +// pass a value <= rg.top. +// If start+n is beyond the limit, then nil values will be copied to the destination slots. +// After the copy, the registry is truncated to be at the end of the copied range, ie the original of the copied values +// are nilled out. (So top will be regv+n) +// CopyRange should ideally be renamed to MoveRange. +func (rg *registry) CopyRange(regv, start, limit, n int) { // +inline-start + newSize := regv + n + // +inline-call rg.checkSize newSize + if limit == -1 || limit > rg.top { + limit = rg.top + } + for i := 0; i < n; i++ { + srcIdx := start + i + if srcIdx >= limit || srcIdx < 0 { + rg.array[regv+i] = LNil + } else { + rg.array[regv+i] = rg.array[srcIdx] + } + } + + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regv + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } +} // +inline-end + +// FillNil fills the registry with nil values from regm to regm+n and then sets the registry top to regm+n +func (rg *registry) FillNil(regm, n int) { // +inline-start + newSize := regm + n + // +inline-call rg.checkSize newSize + for i := 0; i < n; i++ { + rg.array[regm+i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regm + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } +} // +inline-end + +func (rg *registry) Insert(value LValue, reg int) { + top := rg.Top() + if reg >= top { + // +inline-call rg.Set reg value + return + } + top-- + for ; top >= reg; top-- { + // FIXME consider using copy() here if Insert() is called enough + // +inline-call rg.Set top+1 rg.Get(top) + } + // +inline-call rg.Set reg value +} + +func (rg *registry) Set(regi int, vali LValue) { // +inline-start + newSize := regi + 1 + // +inline-call rg.checkSize newSize + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } +} // +inline-end + +func (rg *registry) SetNumber(regi int, vali LNumber) { // +inline-start + newSize := regi + 1 + // +inline-call rg.checkSize newSize + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } +} // +inline-end + +func (rg *registry) IsFull() bool { + return rg.top >= cap(rg.array) +} + +/* }}} */ + +/* Global {{{ */ + +func newGlobal() *Global { + return &Global{ + MainThread: nil, + Registry: newLTable(0, 32), + Global: newLTable(0, 64), + builtinMts: make(map[int]LValue), + tempFiles: make([]*os.File, 0, 10), + } +} + +/* }}} */ + +/* package local methods {{{ */ + +func panicWithTraceback(L *LState) { + err := newApiError(ApiErrorRun, L.Get(-1)) + err.StackTrace = L.stackTrace(0) + panic(err) +} + +func panicWithoutTraceback(L *LState) { + err := newApiError(ApiErrorRun, L.Get(-1)) + panic(err) +} + +func newLState(options Options) *LState { + al := newAllocator(32) + ls := &LState{ + G: newGlobal(), + Parent: nil, + Panic: panicWithTraceback, + Dead: false, + Options: options, + + stop: 0, + alloc: al, + currentFrame: nil, + wrapped: false, + uvcache: nil, + hasErrorFunc: false, + mainLoop: mainLoop, + ctx: nil, + } + if options.MinimizeStackMemory { + ls.stack = newAutoGrowingCallFrameStack(options.CallStackSize) + } else { + ls.stack = newFixedCallFrameStack(options.CallStackSize) + } + ls.reg = newRegistry(ls, options.RegistrySize, options.RegistryGrowStep, options.RegistryMaxSize, al) + ls.Env = ls.G.Global + return ls +} + +func (ls *LState) printReg() { + println("-------------------------") + println("thread:", ls) + println("top:", ls.reg.Top()) + if ls.currentFrame != nil { + println("function base:", ls.currentFrame.Base) + println("return base:", ls.currentFrame.ReturnBase) + } else { + println("(vm not started)") + } + println("local base:", ls.currentLocalBase()) + for i := 0; i < ls.reg.Top(); i++ { + println(i, ls.reg.Get(i).String()) + } + println("-------------------------") +} + +func (ls *LState) printCallStack() { + println("-------------------------") + for i := 0; i < ls.stack.Sp(); i++ { + print(i) + print(" ") + frame := ls.stack.At(i) + if frame == nil { + break + } + if frame.Fn.IsG { + println("IsG:", true, "Frame:", frame, "Fn:", frame.Fn) + } else { + println("IsG:", false, "Frame:", frame, "Fn:", frame.Fn, "pc:", frame.Pc) + } + } + println("-------------------------") +} + +func (ls *LState) closeAllUpvalues() { // +inline-start + for cf := ls.currentFrame; cf != nil; cf = cf.Parent { + if !cf.Fn.IsG { + ls.closeUpvalues(cf.LocalBase) + } + } +} // +inline-end + +func (ls *LState) raiseError(level int, format string, args ...interface{}) { + if !ls.hasErrorFunc { + ls.closeAllUpvalues() + } + message := format + if len(args) > 0 { + message = fmt.Sprintf(format, args...) + } + if level > 0 { + message = fmt.Sprintf("%v %v", ls.where(level-1, true), message) + } + if ls.reg.IsFull() { + // if the registry is full then it won't be possible to push a value, in this case, force a larger size + ls.reg.forceResize(ls.reg.Top() + 1) + } + ls.reg.Push(LString(message)) + ls.Panic(ls) +} + +func (ls *LState) findLocal(frame *callFrame, no int) string { + fn := frame.Fn + if !fn.IsG { + if name, ok := fn.LocalName(no, frame.Pc-1); ok { + return name + } + } + var top int + if ls.currentFrame == frame { + top = ls.reg.Top() + } else if frame.Idx+1 < ls.stack.Sp() { + top = ls.stack.At(frame.Idx + 1).Base + } else { + return "" + } + if top-frame.LocalBase >= no { + return "(*temporary)" + } + return "" +} + +func (ls *LState) where(level int, skipg bool) string { + dbg, ok := ls.GetStack(level) + if !ok { + return "" + } + cf := dbg.frame + proto := cf.Fn.Proto + sourcename := "[G]" + if proto != nil { + sourcename = proto.SourceName + } else if skipg { + return ls.where(level+1, skipg) + } + line := "" + if proto != nil { + line = fmt.Sprintf("%v:", proto.DbgSourcePositions[cf.Pc-1]) + } + return fmt.Sprintf("%v:%v", sourcename, line) +} + +func (ls *LState) stackTrace(level int) string { + buf := []string{} + header := "stack traceback:" + if ls.currentFrame != nil { + i := 0 + for dbg, ok := ls.GetStack(i); ok; dbg, ok = ls.GetStack(i) { + cf := dbg.frame + buf = append(buf, fmt.Sprintf("\t%v in %v", ls.Where(i), ls.formattedFrameFuncName(cf))) + if !cf.Fn.IsG && cf.TailCall > 0 { + for tc := cf.TailCall; tc > 0; tc-- { + buf = append(buf, "\t(tailcall): ?") + i++ + } + } + i++ + } + } + buf = append(buf, fmt.Sprintf("\t%v: %v", "[G]", "?")) + buf = buf[intMax(0, intMin(level, len(buf))):len(buf)] + if len(buf) > 20 { + newbuf := make([]string, 0, 20) + newbuf = append(newbuf, buf[0:7]...) + newbuf = append(newbuf, "\t...") + newbuf = append(newbuf, buf[len(buf)-7:len(buf)]...) + buf = newbuf + } + return fmt.Sprintf("%s\n%s", header, strings.Join(buf, "\n")) +} + +func (ls *LState) formattedFrameFuncName(fr *callFrame) string { + name, ischunk := ls.frameFuncName(fr) + if ischunk { + return name + } + if name[0] != '(' && name[0] != '<' { + return fmt.Sprintf("function '%s'", name) + } + return fmt.Sprintf("function %s", name) +} + +func (ls *LState) rawFrameFuncName(fr *callFrame) string { + name, _ := ls.frameFuncName(fr) + return name +} + +func (ls *LState) frameFuncName(fr *callFrame) (string, bool) { + frame := fr.Parent + if frame == nil { + if ls.Parent == nil { + return "main chunk", true + } else { + return "corountine", true + } + } + if !frame.Fn.IsG { + pc := frame.Pc - 1 + for _, call := range frame.Fn.Proto.DbgCalls { + if call.Pc == pc { + name := call.Name + if (name == "?" || fr.TailCall > 0) && !fr.Fn.IsG { + name = fmt.Sprintf("<%v:%v>", fr.Fn.Proto.SourceName, fr.Fn.Proto.LineDefined) + } + return name, false + } + } + } + if !fr.Fn.IsG { + return fmt.Sprintf("<%v:%v>", fr.Fn.Proto.SourceName, fr.Fn.Proto.LineDefined), false + } + return "(anonymous)", false +} + +func (ls *LState) isStarted() bool { + return ls.currentFrame != nil +} + +func (ls *LState) kill() { + ls.Dead = true + if ls.ctxCancelFn != nil { + ls.ctxCancelFn() + } +} + +func (ls *LState) indexToReg(idx int) int { + base := ls.currentLocalBase() + if idx > 0 { + return base + idx - 1 + } else if idx == 0 { + return -1 + } else { + tidx := ls.reg.Top() + idx + if tidx < base { + return -1 + } + return tidx + } +} + +func (ls *LState) currentLocalBase() int { + base := 0 + if ls.currentFrame != nil { + base = ls.currentFrame.LocalBase + } + return base +} + +func (ls *LState) currentEnv() *LTable { + return ls.Env + /* + if ls.currentFrame == nil { + return ls.Env + } + return ls.currentFrame.Fn.Env + */ +} + +func (ls *LState) rkValue(idx int) LValue { + /* + if OpIsK(idx) { + return ls.currentFrame.Fn.Proto.Constants[opIndexK(idx)] + } + return ls.reg.Get(ls.currentFrame.LocalBase + idx) + */ + if (idx & opBitRk) != 0 { + return ls.currentFrame.Fn.Proto.Constants[idx & ^opBitRk] + } + return ls.reg.array[ls.currentFrame.LocalBase+idx] +} + +func (ls *LState) rkString(idx int) string { + if (idx & opBitRk) != 0 { + return ls.currentFrame.Fn.Proto.stringConstants[idx & ^opBitRk] + } + return string(ls.reg.array[ls.currentFrame.LocalBase+idx].(LString)) +} + +func (ls *LState) closeUpvalues(idx int) { // +inline-start + if ls.uvcache != nil { + var prev *Upvalue + for uv := ls.uvcache; uv != nil; uv = uv.next { + if uv.index >= idx { + if prev != nil { + prev.next = nil + } else { + ls.uvcache = nil + } + uv.Close() + } + prev = uv + } + } +} // +inline-end + +func (ls *LState) findUpvalue(idx int) *Upvalue { + var prev *Upvalue + var next *Upvalue + if ls.uvcache != nil { + for uv := ls.uvcache; uv != nil; uv = uv.next { + if uv.index == idx { + return uv + } + if uv.index > idx { + next = uv + break + } + prev = uv + } + } + uv := &Upvalue{reg: ls.reg, index: idx, closed: false} + if prev != nil { + prev.next = uv + } else { + ls.uvcache = uv + } + if next != nil { + uv.next = next + } + return uv +} + +func (ls *LState) metatable(lvalue LValue, rawget bool) LValue { + var metatable LValue = LNil + switch obj := lvalue.(type) { + case *LTable: + metatable = obj.Metatable + case *LUserData: + metatable = obj.Metatable + default: + if table, ok := ls.G.builtinMts[int(obj.Type())]; ok { + metatable = table + } + } + + if !rawget && metatable != LNil { + oldmt := metatable + if tb, ok := metatable.(*LTable); ok { + metatable = tb.RawGetString("__metatable") + if metatable == LNil { + metatable = oldmt + } + } + } + + return metatable +} + +func (ls *LState) metaOp1(lvalue LValue, event string) LValue { + if mt := ls.metatable(lvalue, true); mt != LNil { + if tb, ok := mt.(*LTable); ok { + return tb.RawGetString(event) + } + } + return LNil +} + +func (ls *LState) metaOp2(value1, value2 LValue, event string) LValue { + if mt := ls.metatable(value1, true); mt != LNil { + if tb, ok := mt.(*LTable); ok { + if ret := tb.RawGetString(event); ret != LNil { + return ret + } + } + } + if mt := ls.metatable(value2, true); mt != LNil { + if tb, ok := mt.(*LTable); ok { + return tb.RawGetString(event) + } + } + return LNil +} + +func (ls *LState) metaCall(lvalue LValue) (*LFunction, bool) { + if fn, ok := lvalue.(*LFunction); ok { + return fn, false + } + if fn, ok := ls.metaOp1(lvalue, "__call").(*LFunction); ok { + return fn, true + } + return nil, false +} + +func (ls *LState) initCallFrame(cf *callFrame) { // +inline-start + if cf.Fn.IsG { + ls.reg.SetTop(cf.LocalBase + cf.NArgs) + } else { + proto := cf.Fn.Proto + nargs := cf.NArgs + np := int(proto.NumParameters) + if nargs < np { + // default any missing arguments to nil + newSize := cf.LocalBase + np + // +inline-call ls.reg.checkSize newSize + for i := nargs; i < np; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + nargs = np + ls.reg.top = newSize + } + + if (proto.IsVarArg & VarArgIsVarArg) == 0 { + if nargs < int(proto.NumUsedRegisters) { + nargs = int(proto.NumUsedRegisters) + } + newSize := cf.LocalBase + nargs + // +inline-call ls.reg.checkSize newSize + for i := np; i < nargs; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + ls.reg.top = cf.LocalBase + int(proto.NumUsedRegisters) + } else { + /* swap vararg positions: + closure + namedparam1 <- lbase + namedparam2 + vararg1 + vararg2 + + TO + + closure + nil + nil + vararg1 + vararg2 + namedparam1 <- lbase + namedparam2 + */ + nvarargs := nargs - np + if nvarargs < 0 { + nvarargs = 0 + } + + ls.reg.SetTop(cf.LocalBase + nargs + np) + for i := 0; i < np; i++ { + //ls.reg.Set(cf.LocalBase+nargs+i, ls.reg.Get(cf.LocalBase+i)) + ls.reg.array[cf.LocalBase+nargs+i] = ls.reg.array[cf.LocalBase+i] + //ls.reg.Set(cf.LocalBase+i, LNil) + ls.reg.array[cf.LocalBase+i] = LNil + } + + if CompatVarArg { + ls.reg.SetTop(cf.LocalBase + nargs + np + 1) + if (proto.IsVarArg & VarArgNeedsArg) != 0 { + argtb := newLTable(nvarargs, 0) + for i := 0; i < nvarargs; i++ { + argtb.RawSetInt(i+1, ls.reg.Get(cf.LocalBase+np+i)) + } + argtb.RawSetString("n", LNumber(nvarargs)) + //ls.reg.Set(cf.LocalBase+nargs+np, argtb) + ls.reg.array[cf.LocalBase+nargs+np] = argtb + } else { + ls.reg.array[cf.LocalBase+nargs+np] = LNil + } + } + cf.LocalBase += nargs + maxreg := cf.LocalBase + int(proto.NumUsedRegisters) + ls.reg.SetTop(maxreg) + } + } +} // +inline-end + +func (ls *LState) pushCallFrame(cf callFrame, fn LValue, meta bool) { // +inline-start + if meta { + cf.NArgs++ + ls.reg.Insert(fn, cf.LocalBase) + } + if cf.Fn == nil { + ls.RaiseError("attempt to call a non-function object") + } + if ls.stack.IsFull() { + ls.RaiseError("stack overflow") + } + ls.stack.Push(cf) + newcf := ls.stack.Last() + // +inline-call ls.initCallFrame newcf + ls.currentFrame = newcf +} // +inline-end + +func (ls *LState) callR(nargs, nret, rbase int) { + base := ls.reg.Top() - nargs - 1 + if rbase < 0 { + rbase = base + } + lv := ls.reg.Get(base) + fn, meta := ls.metaCall(lv) + ls.pushCallFrame(callFrame{ + Fn: fn, + Pc: 0, + Base: base, + LocalBase: base + 1, + ReturnBase: rbase, + NArgs: nargs, + NRet: nret, + Parent: ls.currentFrame, + TailCall: 0, + }, lv, meta) + if ls.G.MainThread == nil { + ls.G.MainThread = ls + ls.G.CurrentThread = ls + ls.mainLoop(ls, nil) + } else { + ls.mainLoop(ls, ls.currentFrame) + } + if nret != MultRet { + ls.reg.SetTop(rbase + nret) + } +} + +func (ls *LState) getField(obj LValue, key LValue) LValue { + curobj := obj + for i := 0; i < MaxTableGetLoop; i++ { + tb, istable := curobj.(*LTable) + if istable { + ret := tb.RawGet(key) + if ret != LNil { + return ret + } + } + metaindex := ls.metaOp1(curobj, "__index") + if metaindex == LNil { + if !istable { + ls.RaiseError("attempt to index a non-table object(%v) with key '%s'", curobj.Type().String(), key.String()) + } + return LNil + } + if metaindex.Type() == LTFunction { + ls.reg.Push(metaindex) + ls.reg.Push(curobj) + ls.reg.Push(key) + ls.Call(2, 1) + return ls.reg.Pop() + } else { + curobj = metaindex + } + } + ls.RaiseError("too many recursions in gettable") + return nil +} + +func (ls *LState) getFieldString(obj LValue, key string) LValue { + curobj := obj + for i := 0; i < MaxTableGetLoop; i++ { + tb, istable := curobj.(*LTable) + if istable { + ret := tb.RawGetString(key) + if ret != LNil { + return ret + } + } + metaindex := ls.metaOp1(curobj, "__index") + if metaindex == LNil { + if !istable { + ls.RaiseError("attempt to index a non-table object(%v) with key '%s'", curobj.Type().String(), key) + } + return LNil + } + if metaindex.Type() == LTFunction { + ls.reg.Push(metaindex) + ls.reg.Push(curobj) + ls.reg.Push(LString(key)) + ls.Call(2, 1) + return ls.reg.Pop() + } else { + curobj = metaindex + } + } + ls.RaiseError("too many recursions in gettable") + return nil +} + +func (ls *LState) setField(obj LValue, key LValue, value LValue) { + curobj := obj + for i := 0; i < MaxTableGetLoop; i++ { + tb, istable := curobj.(*LTable) + if istable { + if tb.RawGet(key) != LNil { + ls.RawSet(tb, key, value) + return + } + } + metaindex := ls.metaOp1(curobj, "__newindex") + if metaindex == LNil { + if !istable { + ls.RaiseError("attempt to index a non-table object(%v) with key '%s'", curobj.Type().String(), key.String()) + } + ls.RawSet(tb, key, value) + return + } + if metaindex.Type() == LTFunction { + ls.reg.Push(metaindex) + ls.reg.Push(curobj) + ls.reg.Push(key) + ls.reg.Push(value) + ls.Call(3, 0) + return + } else { + curobj = metaindex + } + } + ls.RaiseError("too many recursions in settable") +} + +func (ls *LState) setFieldString(obj LValue, key string, value LValue) { + curobj := obj + for i := 0; i < MaxTableGetLoop; i++ { + tb, istable := curobj.(*LTable) + if istable { + if tb.RawGetString(key) != LNil { + tb.RawSetString(key, value) + return + } + } + metaindex := ls.metaOp1(curobj, "__newindex") + if metaindex == LNil { + if !istable { + ls.RaiseError("attempt to index a non-table object(%v) with key '%s'", curobj.Type().String(), key) + } + tb.RawSetString(key, value) + return + } + if metaindex.Type() == LTFunction { + ls.reg.Push(metaindex) + ls.reg.Push(curobj) + ls.reg.Push(LString(key)) + ls.reg.Push(value) + ls.Call(3, 0) + return + } else { + curobj = metaindex + } + } + ls.RaiseError("too many recursions in settable") +} + +/* }}} */ + +/* api methods {{{ */ + +func NewState(opts ...Options) *LState { + var ls *LState + if len(opts) == 0 { + ls = newLState(Options{ + CallStackSize: CallStackSize, + RegistrySize: RegistrySize, + }) + ls.OpenLibs() + } else { + if opts[0].CallStackSize < 1 { + opts[0].CallStackSize = CallStackSize + } + if opts[0].RegistrySize < 128 { + opts[0].RegistrySize = RegistrySize + } + if opts[0].RegistryMaxSize < opts[0].RegistrySize { + opts[0].RegistryMaxSize = 0 // disable growth if max size is smaller than initial size + } else { + // if growth enabled, grow step is set + if opts[0].RegistryGrowStep < 1 { + opts[0].RegistryGrowStep = RegistryGrowStep + } + } + ls = newLState(opts[0]) + if !opts[0].SkipOpenLibs { + ls.OpenLibs() + } + } + return ls +} + +func (ls *LState) IsClosed() bool { + return ls.stack == nil +} + +func (ls *LState) Close() { + atomic.AddInt32(&ls.stop, 1) + for _, file := range ls.G.tempFiles { + // ignore errors in these operations + file.Close() + os.Remove(file.Name()) + } + ls.stack.FreeAll() + ls.stack = nil +} + +/* registry operations {{{ */ + +func (ls *LState) GetTop() int { + return ls.reg.Top() - ls.currentLocalBase() +} + +func (ls *LState) SetTop(idx int) { + base := ls.currentLocalBase() + newtop := ls.indexToReg(idx) + 1 + if newtop < base { + ls.reg.SetTop(base) + } else { + ls.reg.SetTop(newtop) + } +} + +func (ls *LState) Replace(idx int, value LValue) { + base := ls.currentLocalBase() + if idx > 0 { + reg := base + idx - 1 + if reg < ls.reg.Top() { + ls.reg.Set(reg, value) + } + } else if idx == 0 { + } else if idx > RegistryIndex { + if tidx := ls.reg.Top() + idx; tidx >= base { + ls.reg.Set(tidx, value) + } + } else { + switch idx { + case RegistryIndex: + if tb, ok := value.(*LTable); ok { + ls.G.Registry = tb + } else { + ls.RaiseError("registry must be a table(%v)", value.Type().String()) + } + case EnvironIndex: + if ls.currentFrame == nil { + ls.RaiseError("no calling environment") + } + if tb, ok := value.(*LTable); ok { + ls.currentFrame.Fn.Env = tb + } else { + ls.RaiseError("environment must be a table(%v)", value.Type().String()) + } + case GlobalsIndex: + if tb, ok := value.(*LTable); ok { + ls.G.Global = tb + } else { + ls.RaiseError("_G must be a table(%v)", value.Type().String()) + } + default: + fn := ls.currentFrame.Fn + index := GlobalsIndex - idx - 1 + if index < len(fn.Upvalues) { + fn.Upvalues[index].SetValue(value) + } + } + } +} + +func (ls *LState) Get(idx int) LValue { + base := ls.currentLocalBase() + if idx > 0 { + reg := base + idx - 1 + if reg < ls.reg.Top() { + return ls.reg.Get(reg) + } + return LNil + } else if idx == 0 { + return LNil + } else if idx > RegistryIndex { + tidx := ls.reg.Top() + idx + if tidx < base { + return LNil + } + return ls.reg.Get(tidx) + } else { + switch idx { + case RegistryIndex: + return ls.G.Registry + case EnvironIndex: + if ls.currentFrame == nil { + return ls.Env + } + return ls.currentFrame.Fn.Env + case GlobalsIndex: + return ls.G.Global + default: + fn := ls.currentFrame.Fn + index := GlobalsIndex - idx - 1 + if index < len(fn.Upvalues) { + return fn.Upvalues[index].Value() + } + return LNil + } + } + return LNil +} + +func (ls *LState) Push(value LValue) { + ls.reg.Push(value) +} + +func (ls *LState) Pop(n int) { + for i := 0; i < n; i++ { + if ls.GetTop() == 0 { + ls.RaiseError("register underflow") + } + ls.reg.Pop() + } +} + +func (ls *LState) Insert(value LValue, index int) { + reg := ls.indexToReg(index) + top := ls.reg.Top() + if reg >= top { + ls.reg.Set(reg, value) + return + } + if reg <= ls.currentLocalBase() { + reg = ls.currentLocalBase() + } + top-- + for ; top >= reg; top-- { + ls.reg.Set(top+1, ls.reg.Get(top)) + } + ls.reg.Set(reg, value) +} + +func (ls *LState) Remove(index int) { + reg := ls.indexToReg(index) + top := ls.reg.Top() + switch { + case reg >= top: + return + case reg < ls.currentLocalBase(): + return + case reg == top-1: + ls.Pop(1) + return + } + for i := reg; i < top-1; i++ { + ls.reg.Set(i, ls.reg.Get(i+1)) + } + ls.reg.SetTop(top - 1) +} + +/* }}} */ + +/* object allocation {{{ */ + +func (ls *LState) NewTable() *LTable { + return newLTable(defaultArrayCap, defaultHashCap) +} + +func (ls *LState) CreateTable(acap, hcap int) *LTable { + return newLTable(acap, hcap) +} + +// NewThread returns a new LState that shares with the original state all global objects. +// If the original state has context.Context, the new state has a new child context of the original state and this function returns its cancel function. +func (ls *LState) NewThread() (*LState, context.CancelFunc) { + thread := newLState(ls.Options) + thread.G = ls.G + thread.Env = ls.Env + var f context.CancelFunc = nil + if ls.ctx != nil { + thread.mainLoop = mainLoopWithContext + thread.ctx, f = context.WithCancel(ls.ctx) + thread.ctxCancelFn = f + } + return thread, f +} + +func (ls *LState) NewFunctionFromProto(proto *FunctionProto) *LFunction { + return newLFunctionL(proto, ls.Env, int(proto.NumUpvalues)) +} + +func (ls *LState) NewUserData() *LUserData { + return &LUserData{ + Env: ls.currentEnv(), + Metatable: LNil, + } +} + +func (ls *LState) NewFunction(fn LGFunction) *LFunction { + return newLFunctionG(fn, ls.currentEnv(), 0) +} + +func (ls *LState) NewClosure(fn LGFunction, upvalues ...LValue) *LFunction { + cl := newLFunctionG(fn, ls.currentEnv(), len(upvalues)) + for i, lv := range upvalues { + cl.Upvalues[i] = &Upvalue{} + cl.Upvalues[i].Close() + cl.Upvalues[i].SetValue(lv) + } + return cl +} + +/* }}} */ + +/* toType {{{ */ + +func (ls *LState) ToBool(n int) bool { + return LVAsBool(ls.Get(n)) +} + +func (ls *LState) ToInt(n int) int { + if lv, ok := ls.Get(n).(LNumber); ok { + return int(lv) + } + if lv, ok := ls.Get(n).(LString); ok { + if num, err := parseNumber(string(lv)); err == nil { + return int(num) + } + } + return 0 +} + +func (ls *LState) ToInt64(n int) int64 { + if lv, ok := ls.Get(n).(LNumber); ok { + return int64(lv) + } + if lv, ok := ls.Get(n).(LString); ok { + if num, err := parseNumber(string(lv)); err == nil { + return int64(num) + } + } + return 0 +} + +func (ls *LState) ToNumber(n int) LNumber { + return LVAsNumber(ls.Get(n)) +} + +func (ls *LState) ToString(n int) string { + return LVAsString(ls.Get(n)) +} + +func (ls *LState) ToTable(n int) *LTable { + if lv, ok := ls.Get(n).(*LTable); ok { + return lv + } + return nil +} + +func (ls *LState) ToFunction(n int) *LFunction { + if lv, ok := ls.Get(n).(*LFunction); ok { + return lv + } + return nil +} + +func (ls *LState) ToUserData(n int) *LUserData { + if lv, ok := ls.Get(n).(*LUserData); ok { + return lv + } + return nil +} + +func (ls *LState) ToThread(n int) *LState { + if lv, ok := ls.Get(n).(*LState); ok { + return lv + } + return nil +} + +/* }}} */ + +/* error & debug operations {{{ */ + +func (ls *LState) registryOverflow() { + ls.RaiseError("registry overflow") +} + +// This function is equivalent to luaL_error( http://www.lua.org/manual/5.1/manual.html#luaL_error ). +func (ls *LState) RaiseError(format string, args ...interface{}) { + ls.raiseError(1, format, args...) +} + +// This function is equivalent to lua_error( http://www.lua.org/manual/5.1/manual.html#lua_error ). +func (ls *LState) Error(lv LValue, level int) { + if str, ok := lv.(LString); ok { + ls.raiseError(level, string(str)) + } else { + if !ls.hasErrorFunc { + ls.closeAllUpvalues() + } + ls.Push(lv) + ls.Panic(ls) + } +} + +func (ls *LState) GetInfo(what string, dbg *Debug, fn LValue) (LValue, error) { + if !strings.HasPrefix(what, ">") { + fn = dbg.frame.Fn + } else { + what = what[1:] + } + f, ok := fn.(*LFunction) + if !ok { + return LNil, newApiErrorS(ApiErrorRun, "can not get debug info(an object in not a function)") + } + + retfn := false + for _, c := range what { + switch c { + case 'f': + retfn = true + case 'S': + if dbg.frame != nil && dbg.frame.Parent == nil { + dbg.What = "main" + } else if f.IsG { + dbg.What = "G" + } else if dbg.frame != nil && dbg.frame.TailCall > 0 { + dbg.What = "tail" + } else { + dbg.What = "Lua" + } + if !f.IsG { + dbg.Source = f.Proto.SourceName + dbg.LineDefined = f.Proto.LineDefined + dbg.LastLineDefined = f.Proto.LastLineDefined + } + case 'l': + if !f.IsG && dbg.frame != nil { + if dbg.frame.Pc > 0 { + dbg.CurrentLine = f.Proto.DbgSourcePositions[dbg.frame.Pc-1] + } + } else { + dbg.CurrentLine = -1 + } + case 'u': + dbg.NUpvalues = len(f.Upvalues) + case 'n': + if dbg.frame != nil { + dbg.Name = ls.rawFrameFuncName(dbg.frame) + } + default: + return LNil, newApiErrorS(ApiErrorRun, "invalid what: "+string(c)) + } + } + + if retfn { + return f, nil + } + return LNil, nil + +} + +func (ls *LState) GetStack(level int) (*Debug, bool) { + frame := ls.currentFrame + for ; level > 0 && frame != nil; frame = frame.Parent { + level-- + if !frame.Fn.IsG { + level -= frame.TailCall + } + } + + if level == 0 && frame != nil { + return &Debug{frame: frame}, true + } else if level < 0 && ls.stack.Sp() > 0 { + return &Debug{frame: ls.stack.At(0)}, true + } + return &Debug{}, false +} + +func (ls *LState) GetLocal(dbg *Debug, no int) (string, LValue) { + frame := dbg.frame + if name := ls.findLocal(frame, no); len(name) > 0 { + return name, ls.reg.Get(frame.LocalBase + no - 1) + } + return "", LNil +} + +func (ls *LState) SetLocal(dbg *Debug, no int, lv LValue) string { + frame := dbg.frame + if name := ls.findLocal(frame, no); len(name) > 0 { + ls.reg.Set(frame.LocalBase+no-1, lv) + return name + } + return "" +} + +func (ls *LState) GetUpvalue(fn *LFunction, no int) (string, LValue) { + if fn.IsG { + return "", LNil + } + + no-- + if no >= 0 && no < len(fn.Upvalues) { + return fn.Proto.DbgUpvalues[no], fn.Upvalues[no].Value() + } + return "", LNil +} + +func (ls *LState) SetUpvalue(fn *LFunction, no int, lv LValue) string { + if fn.IsG { + return "" + } + + no-- + if no >= 0 && no < len(fn.Upvalues) { + fn.Upvalues[no].SetValue(lv) + return fn.Proto.DbgUpvalues[no] + } + return "" +} + +/* }}} */ + +/* env operations {{{ */ + +func (ls *LState) GetFEnv(obj LValue) LValue { + switch lv := obj.(type) { + case *LFunction: + return lv.Env + case *LUserData: + return lv.Env + case *LState: + return lv.Env + } + return LNil +} + +func (ls *LState) SetFEnv(obj LValue, env LValue) { + tb, ok := env.(*LTable) + if !ok { + ls.RaiseError("cannot use %v as an environment", env.Type().String()) + } + + switch lv := obj.(type) { + case *LFunction: + lv.Env = tb + case *LUserData: + lv.Env = tb + case *LState: + lv.Env = tb + } + /* do nothing */ +} + +/* }}} */ + +/* table operations {{{ */ + +func (ls *LState) RawGet(tb *LTable, key LValue) LValue { + return tb.RawGet(key) +} + +func (ls *LState) RawGetInt(tb *LTable, key int) LValue { + return tb.RawGetInt(key) +} + +func (ls *LState) GetField(obj LValue, skey string) LValue { + return ls.getFieldString(obj, skey) +} + +func (ls *LState) GetTable(obj LValue, key LValue) LValue { + return ls.getField(obj, key) +} + +func (ls *LState) RawSet(tb *LTable, key LValue, value LValue) { + if n, ok := key.(LNumber); ok && math.IsNaN(float64(n)) { + ls.RaiseError("table index is NaN") + } else if key == LNil { + ls.RaiseError("table index is nil") + } + tb.RawSet(key, value) +} + +func (ls *LState) RawSetInt(tb *LTable, key int, value LValue) { + tb.RawSetInt(key, value) +} + +func (ls *LState) SetField(obj LValue, key string, value LValue) { + ls.setFieldString(obj, key, value) +} + +func (ls *LState) SetTable(obj LValue, key LValue, value LValue) { + ls.setField(obj, key, value) +} + +func (ls *LState) ForEach(tb *LTable, cb func(LValue, LValue)) { + tb.ForEach(cb) +} + +func (ls *LState) GetGlobal(name string) LValue { + return ls.GetField(ls.Get(GlobalsIndex), name) +} + +func (ls *LState) SetGlobal(name string, value LValue) { + ls.SetField(ls.Get(GlobalsIndex), name, value) +} + +func (ls *LState) Next(tb *LTable, key LValue) (LValue, LValue) { + return tb.Next(key) +} + +/* }}} */ + +/* unary operations {{{ */ + +func (ls *LState) ObjLen(v1 LValue) int { + if v1.Type() == LTString { + return len(string(v1.(LString))) + } + op := ls.metaOp1(v1, "__len") + if op.Type() == LTFunction { + ls.Push(op) + ls.Push(v1) + ls.Call(1, 1) + ret := ls.reg.Pop() + if ret.Type() == LTNumber { + return int(ret.(LNumber)) + } + } else if v1.Type() == LTTable { + return v1.(*LTable).Len() + } + return 0 +} + +/* }}} */ + +/* binary operations {{{ */ + +func (ls *LState) Concat(values ...LValue) string { + top := ls.reg.Top() + for _, value := range values { + ls.reg.Push(value) + } + ret := stringConcat(ls, len(values), ls.reg.Top()-1) + ls.reg.SetTop(top) + return LVAsString(ret) +} + +func (ls *LState) LessThan(lhs, rhs LValue) bool { + return lessThan(ls, lhs, rhs) +} + +func (ls *LState) Equal(lhs, rhs LValue) bool { + return equals(ls, lhs, rhs, false) +} + +func (ls *LState) RawEqual(lhs, rhs LValue) bool { + return equals(ls, lhs, rhs, true) +} + +/* }}} */ + +/* register operations {{{ */ + +func (ls *LState) Register(name string, fn LGFunction) { + ls.SetGlobal(name, ls.NewFunction(fn)) +} + +/* }}} */ + +/* load and function call operations {{{ */ + +func (ls *LState) Load(reader io.Reader, name string) (*LFunction, error) { + chunk, err := parse.Parse(reader, name) + if err != nil { + return nil, newApiErrorE(ApiErrorSyntax, err) + } + proto, err := Compile(chunk, name) + if err != nil { + return nil, newApiErrorE(ApiErrorSyntax, err) + } + return newLFunctionL(proto, ls.currentEnv(), 0), nil +} + +func (ls *LState) Call(nargs, nret int) { + ls.callR(nargs, nret, -1) +} + +func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) { + err = nil + sp := ls.stack.Sp() + base := ls.reg.Top() - nargs - 1 + oldpanic := ls.Panic + ls.Panic = panicWithoutTraceback + if errfunc != nil { + ls.hasErrorFunc = true + } + defer func() { + ls.Panic = oldpanic + ls.hasErrorFunc = false + rcv := recover() + if rcv != nil { + if _, ok := rcv.(*ApiError); !ok { + err = newApiErrorS(ApiErrorPanic, fmt.Sprint(rcv)) + if ls.Options.IncludeGoStackTrace { + buf := make([]byte, 4096) + runtime.Stack(buf, false) + err.(*ApiError).StackTrace = strings.Trim(string(buf), "\000") + "\n" + ls.stackTrace(0) + } + } else { + err = rcv.(*ApiError) + } + if errfunc != nil { + ls.Push(errfunc) + ls.Push(err.(*ApiError).Object) + ls.Panic = panicWithoutTraceback + defer func() { + ls.Panic = oldpanic + rcv := recover() + if rcv != nil { + if _, ok := rcv.(*ApiError); !ok { + err = newApiErrorS(ApiErrorPanic, fmt.Sprint(rcv)) + if ls.Options.IncludeGoStackTrace { + buf := make([]byte, 4096) + runtime.Stack(buf, false) + err.(*ApiError).StackTrace = strings.Trim(string(buf), "\000") + ls.stackTrace(0) + } + } else { + err = rcv.(*ApiError) + err.(*ApiError).StackTrace = ls.stackTrace(0) + } + ls.stack.SetSp(sp) + ls.currentFrame = ls.stack.Last() + ls.reg.SetTop(base) + } + }() + ls.Call(1, 1) + err = newApiError(ApiErrorError, ls.Get(-1)) + } else if len(err.(*ApiError).StackTrace) == 0 { + err.(*ApiError).StackTrace = ls.stackTrace(0) + } + ls.stack.SetSp(sp) + ls.currentFrame = ls.stack.Last() + ls.reg.SetTop(base) + } + ls.stack.SetSp(sp) + if sp == 0 { + ls.currentFrame = nil + } + }() + + ls.Call(nargs, nret) + + return +} + +func (ls *LState) GPCall(fn LGFunction, data LValue) error { + ls.Push(newLFunctionG(fn, ls.currentEnv(), 0)) + ls.Push(data) + return ls.PCall(1, MultRet, nil) +} + +func (ls *LState) CallByParam(cp P, args ...LValue) error { + ls.Push(cp.Fn) + for _, arg := range args { + ls.Push(arg) + } + + if cp.Protect { + return ls.PCall(len(args), cp.NRet, cp.Handler) + } + ls.Call(len(args), cp.NRet) + return nil +} + +/* }}} */ + +/* metatable operations {{{ */ + +func (ls *LState) GetMetatable(obj LValue) LValue { + return ls.metatable(obj, false) +} + +func (ls *LState) SetMetatable(obj LValue, mt LValue) { + switch mt.(type) { + case *LNilType, *LTable: + default: + ls.RaiseError("metatable must be a table or nil, but got %v", mt.Type().String()) + } + + switch v := obj.(type) { + case *LTable: + v.Metatable = mt + case *LUserData: + v.Metatable = mt + default: + ls.G.builtinMts[int(obj.Type())] = mt + } +} + +/* }}} */ + +/* coroutine operations {{{ */ + +func (ls *LState) Status(th *LState) string { + status := "suspended" + if th.Dead { + status = "dead" + } else if ls.G.CurrentThread == th { + status = "running" + } else if ls.Parent == th { + status = "normal" + } + return status +} + +func (ls *LState) Resume(th *LState, fn *LFunction, args ...LValue) (ResumeState, error, []LValue) { + isstarted := th.isStarted() + if !isstarted { + base := 0 + th.stack.Push(callFrame{ + Fn: fn, + Pc: 0, + Base: base, + LocalBase: base + 1, + ReturnBase: base, + NArgs: 0, + NRet: MultRet, + Parent: nil, + TailCall: 0, + }) + } + + if ls.G.CurrentThread == th { + return ResumeError, newApiErrorS(ApiErrorRun, "can not resume a running thread"), nil + } + if th.Dead { + return ResumeError, newApiErrorS(ApiErrorRun, "can not resume a dead thread"), nil + } + th.Parent = ls + ls.G.CurrentThread = th + if !isstarted { + cf := th.stack.Last() + th.currentFrame = cf + th.SetTop(0) + for _, arg := range args { + th.Push(arg) + } + cf.NArgs = len(args) + th.initCallFrame(cf) + th.Panic = panicWithoutTraceback + } else { + for _, arg := range args { + th.Push(arg) + } + } + top := ls.GetTop() + threadRun(th) + haserror := LVIsFalse(ls.Get(top + 1)) + ret := make([]LValue, 0, ls.GetTop()) + for idx := top + 2; idx <= ls.GetTop(); idx++ { + ret = append(ret, ls.Get(idx)) + } + if len(ret) == 0 { + ret = append(ret, LNil) + } + ls.SetTop(top) + + if haserror { + return ResumeError, newApiError(ApiErrorRun, ret[0]), nil + } else if th.stack.IsEmpty() { + return ResumeOK, nil, ret + } + return ResumeYield, nil, ret +} + +func (ls *LState) Yield(values ...LValue) int { + ls.SetTop(0) + for _, lv := range values { + ls.Push(lv) + } + return -1 +} + +func (ls *LState) XMoveTo(other *LState, n int) { + if ls == other { + return + } + top := ls.GetTop() + n = intMin(n, top) + for i := n; i > 0; i-- { + other.Push(ls.Get(top - i + 1)) + } + ls.SetTop(top - n) +} + +/* }}} */ + +/* GopherLua original APIs {{{ */ + +// Set maximum memory size. This function can only be called from the main thread. +func (ls *LState) SetMx(mx int) { + if ls.Parent != nil { + ls.RaiseError("sub threads are not allowed to set a memory limit") + } + go func() { + limit := uint64(mx * 1024 * 1024) //MB + var s runtime.MemStats + for atomic.LoadInt32(&ls.stop) == 0 { + runtime.ReadMemStats(&s) + if s.Alloc >= limit { + fmt.Println("out of memory") + os.Exit(3) + } + time.Sleep(100 * time.Millisecond) + } + }() +} + +// SetContext set a context ctx to this LState. The provided ctx must be non-nil. +func (ls *LState) SetContext(ctx context.Context) { + ls.mainLoop = mainLoopWithContext + ls.ctx = ctx +} + +// Context returns the LState's context. To change the context, use WithContext. +func (ls *LState) Context() context.Context { + return ls.ctx +} + +// RemoveContext removes the context associated with this LState and returns this context. +func (ls *LState) RemoveContext() context.Context { + oldctx := ls.ctx + ls.mainLoop = mainLoop + ls.ctx = nil + return oldctx +} + +// Converts the Lua value at the given acceptable index to the chan LValue. +func (ls *LState) ToChannel(n int) chan LValue { + if lv, ok := ls.Get(n).(LChannel); ok { + return (chan LValue)(lv) + } + return nil +} + +// RemoveCallerFrame removes the stack frame above the current stack frame. This is useful in tail calls. It returns +// the new current frame. +func (ls *LState) RemoveCallerFrame() *callFrame { + cs := ls.stack + sp := cs.Sp() + parentFrame := cs.At(sp - 2) + currentFrame := cs.At(sp - 1) + parentsParentFrame := parentFrame.Parent + *parentFrame = *currentFrame + parentFrame.Parent = parentsParentFrame + parentFrame.Idx = sp - 2 + cs.Pop() + return parentFrame +} + +/* }}} */ + +/* }}} */ + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/_vm.go b/scraper-go/vendor/github.com/yuin/gopher-lua/_vm.go new file mode 100644 index 00000000..ee2be040 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/_vm.go @@ -0,0 +1,1049 @@ +package lua + +import ( + "fmt" + "math" + "strings" +) + +func mainLoop(L *LState, baseframe *callFrame) { + var inst uint32 + var cf *callFrame + + if L.stack.IsEmpty() { + return + } + + L.currentFrame = L.stack.Last() + if L.currentFrame.Fn.IsG { + callGFunction(L, false) + return + } + + for { + cf = L.currentFrame + inst = cf.Fn.Proto.Code[cf.Pc] + cf.Pc++ + if jumpTable[int(inst>>26)](L, inst, baseframe) == 1 { + return + } + } +} + +func mainLoopWithContext(L *LState, baseframe *callFrame) { + var inst uint32 + var cf *callFrame + + if L.stack.IsEmpty() { + return + } + + L.currentFrame = L.stack.Last() + if L.currentFrame.Fn.IsG { + callGFunction(L, false) + return + } + + for { + cf = L.currentFrame + inst = cf.Fn.Proto.Code[cf.Pc] + cf.Pc++ + select { + case <-L.ctx.Done(): + L.RaiseError(L.ctx.Err().Error()) + return + default: + if jumpTable[int(inst>>26)](L, inst, baseframe) == 1 { + return + } + } + } +} + +// regv is the first target register to copy the return values to. +// It can be reg.top, indicating that the copied values are going into new registers, or it can be below reg.top +// Indicating that the values should be within the existing registers. +// b is the available number of return values + 1. +// n is the desired number of return values. +// If n more than the available return values then the extra values are set to nil. +// When this function returns the top of the registry will be set to regv+n. +func copyReturnValues(L *LState, regv, start, n, b int) { // +inline-start + if b == 1 { + // +inline-call L.reg.FillNil regv n + } else { + // +inline-call L.reg.CopyRange regv start -1 n + if b > 1 && n > (b-1) { + // +inline-call L.reg.FillNil regv+b-1 n-(b-1) + } + } +} // +inline-end + +func switchToParentThread(L *LState, nargs int, haserror bool, kill bool) { + parent := L.Parent + if parent == nil { + L.RaiseError("can not yield from outside of a coroutine") + } + L.G.CurrentThread = parent + L.Parent = nil + if !L.wrapped { + if haserror { + parent.Push(LFalse) + } else { + parent.Push(LTrue) + } + } + L.XMoveTo(parent, nargs) + L.stack.Pop() + offset := L.currentFrame.LocalBase - L.currentFrame.ReturnBase + L.currentFrame = L.stack.Last() + L.reg.SetTop(L.reg.Top() - offset) // remove 'yield' function(including tailcalled functions) + if kill { + L.kill() + } +} + +func callGFunction(L *LState, tailcall bool) bool { + frame := L.currentFrame + gfnret := frame.Fn.GFunction(L) + if tailcall { + L.currentFrame = L.RemoveCallerFrame() + } + + if gfnret < 0 { + switchToParentThread(L, L.GetTop(), false, false) + return true + } + + wantret := frame.NRet + if wantret == MultRet { + wantret = gfnret + } + + if tailcall && L.Parent != nil && L.stack.Sp() == 1 { + switchToParentThread(L, wantret, false, true) + return true + } + + // +inline-call L.reg.CopyRange frame.ReturnBase L.reg.Top()-gfnret -1 wantret + L.stack.Pop() + L.currentFrame = L.stack.Last() + return false +} + +func threadRun(L *LState) { + if L.stack.IsEmpty() { + return + } + + defer func() { + if rcv := recover(); rcv != nil { + var lv LValue + if v, ok := rcv.(*ApiError); ok { + lv = v.Object + } else { + lv = LString(fmt.Sprint(rcv)) + } + if parent := L.Parent; parent != nil { + if L.wrapped { + L.Push(lv) + parent.Panic(L) + } else { + L.SetTop(0) + L.Push(lv) + switchToParentThread(L, 1, true, true) + } + } else { + panic(rcv) + } + } + }() + L.mainLoop(L, nil) +} + +type instFunc func(*LState, uint32, *callFrame) int + +var jumpTable [opCodeMax + 1]instFunc + +func init() { + jumpTable = [opCodeMax + 1]instFunc{ + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_MOVE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + v := reg.Get(lbase + B) + // +inline-call reg.Set RA v + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_MOVEN + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + v := reg.Get(lbase + B) + // +inline-call reg.Set lbase+A v + code := cf.Fn.Proto.Code + pc := cf.Pc + for i := 0; i < C; i++ { + inst = code[pc] + pc++ + A = int(inst>>18) & 0xff //GETA + B = int(inst & 0x1ff) //GETB + v := reg.Get(lbase + B) + // +inline-call reg.Set lbase+A v + } + cf.Pc = pc + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LOADK + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Bx := int(inst & 0x3ffff) //GETBX + v := cf.Fn.Proto.Constants[Bx] + // +inline-call reg.Set RA v + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LOADBOOL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + if B != 0 { + // +inline-call reg.Set RA LTrue + } else { + // +inline-call reg.Set RA LFalse + } + if C != 0 { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LOADNIL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + for i := RA; i <= lbase+B; i++ { + // +inline-call reg.Set i LNil + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_GETUPVAL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + v := cf.Fn.Upvalues[B].Value() + // +inline-call reg.Set RA v + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_GETGLOBAL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Bx := int(inst & 0x3ffff) //GETBX + //reg.Set(RA, L.getField(cf.Fn.Env, cf.Fn.Proto.Constants[Bx])) + v := L.getFieldString(cf.Fn.Env, cf.Fn.Proto.stringConstants[Bx]) + // +inline-call reg.Set RA v + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_GETTABLE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + v := L.getField(reg.Get(lbase+B), L.rkValue(C)) + // +inline-call reg.Set RA v + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_GETTABLEKS + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + v := L.getFieldString(reg.Get(lbase+B), L.rkString(C)) + // +inline-call reg.Set RA v + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETGLOBAL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Bx := int(inst & 0x3ffff) //GETBX + //L.setField(cf.Fn.Env, cf.Fn.Proto.Constants[Bx], reg.Get(RA)) + L.setFieldString(cf.Fn.Env, cf.Fn.Proto.stringConstants[Bx], reg.Get(RA)) + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETUPVAL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + cf.Fn.Upvalues[B].SetValue(reg.Get(RA)) + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETTABLE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + L.setField(reg.Get(RA), L.rkValue(B), L.rkValue(C)) + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETTABLEKS + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + L.setFieldString(reg.Get(RA), L.rkString(B), L.rkValue(C)) + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_NEWTABLE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + v := newLTable(B, C) + // +inline-call reg.Set RA v + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SELF + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + selfobj := reg.Get(lbase + B) + v := L.getFieldString(selfobj, L.rkString(C)) + // +inline-call reg.Set RA v + // +inline-call reg.Set RA+1 selfobj + return 0 + }, + opArith, // OP_ADD + opArith, // OP_SUB + opArith, // OP_MUL + opArith, // OP_DIV + opArith, // OP_MOD + opArith, // OP_POW + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_UNM + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + unaryv := L.rkValue(B) + if nm, ok := unaryv.(LNumber); ok { + // +inline-call reg.Set RA -nm + } else { + op := L.metaOp1(unaryv, "__unm") + if op.Type() == LTFunction { + reg.Push(op) + reg.Push(unaryv) + L.Call(1, 1) + // +inline-call reg.Set RA reg.Pop() + } else if str, ok1 := unaryv.(LString); ok1 { + if num, err := parseNumber(string(str)); err == nil { + // +inline-call reg.Set RA -num + } else { + L.RaiseError("__unm undefined") + } + } else { + L.RaiseError("__unm undefined") + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_NOT + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + if LVIsFalse(reg.Get(lbase + B)) { + // +inline-call reg.Set RA LTrue + } else { + // +inline-call reg.Set RA LFalse + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LEN + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + switch lv := L.rkValue(B).(type) { + case LString: + // +inline-call reg.SetNumber RA LNumber(len(lv)) + default: + op := L.metaOp1(lv, "__len") + if op.Type() == LTFunction { + reg.Push(op) + reg.Push(lv) + L.Call(1, 1) + ret := reg.Pop() + if ret.Type() == LTNumber { + v, _ := ret.(LNumber) + // +inline-call reg.SetNumber RA v + } else { + // +inline-call reg.Set RA ret + } + } else if lv.Type() == LTTable { + // +inline-call reg.SetNumber RA LNumber(lv.(*LTable).Len()) + } else { + L.RaiseError("__len undefined") + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_CONCAT + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + RC := lbase + C + RB := lbase + B + v := stringConcat(L, RC-RB+1, RC) + // +inline-call reg.Set RA v + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_JMP + cf := L.currentFrame + Sbx := int(inst&0x3ffff) - opMaxArgSbx //GETSBX + cf.Pc += Sbx + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_EQ + cf := L.currentFrame + A := int(inst>>18) & 0xff //GETA + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + ret := equals(L, L.rkValue(B), L.rkValue(C), false) + v := 1 + if ret { + v = 0 + } + if v == A { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LT + cf := L.currentFrame + A := int(inst>>18) & 0xff //GETA + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + ret := lessThan(L, L.rkValue(B), L.rkValue(C)) + v := 1 + if ret { + v = 0 + } + if v == A { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LE + cf := L.currentFrame + A := int(inst>>18) & 0xff //GETA + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + lhs := L.rkValue(B) + rhs := L.rkValue(C) + ret := false + + if v1, ok1 := lhs.(LNumber); ok1 { + if v2, ok2 := rhs.(LNumber); ok2 { + ret = v1 <= v2 + } else { + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + } + } else { + if lhs.Type() != rhs.Type() { + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + } + switch lhs.Type() { + case LTString: + ret = strCmp(string(lhs.(LString)), string(rhs.(LString))) <= 0 + default: + switch objectRational(L, lhs, rhs, "__le") { + case 1: + ret = true + case 0: + ret = false + default: + ret = !objectRationalWithError(L, rhs, lhs, "__lt") + } + } + } + + v := 1 + if ret { + v = 0 + } + if v == A { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_TEST + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + C := int(inst>>9) & 0x1ff //GETC + if LVAsBool(reg.Get(RA)) == (C == 0) { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_TESTSET + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + if value := reg.Get(lbase + B); LVAsBool(value) != (C == 0) { + // +inline-call reg.Set RA value + } else { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_CALL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + nargs := B - 1 + if B == 0 { + nargs = reg.Top() - (RA + 1) + } + lv := reg.Get(RA) + nret := C - 1 + var callable *LFunction + var meta bool + if fn, ok := lv.(*LFunction); ok { + callable = fn + meta = false + } else { + callable, meta = L.metaCall(lv) + } + // +inline-call L.pushCallFrame callFrame{Fn:callable,Pc:0,Base:RA,LocalBase:RA+1,ReturnBase:RA,NArgs:nargs,NRet:nret,Parent:cf,TailCall:0} lv meta + if callable.IsG && callGFunction(L, false) { + return 1 + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_TAILCALL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + nargs := B - 1 + if B == 0 { + nargs = reg.Top() - (RA + 1) + } + lv := reg.Get(RA) + var callable *LFunction + var meta bool + if fn, ok := lv.(*LFunction); ok { + callable = fn + meta = false + } else { + callable, meta = L.metaCall(lv) + } + if callable == nil { + L.RaiseError("attempt to call a non-function object") + } + // +inline-call L.closeUpvalues lbase + if callable.IsG { + luaframe := cf + L.pushCallFrame(callFrame{ + Fn: callable, + Pc: 0, + Base: RA, + LocalBase: RA + 1, + ReturnBase: cf.ReturnBase, + NArgs: nargs, + NRet: cf.NRet, + Parent: cf, + TailCall: 0, + }, lv, meta) + if callGFunction(L, true) { + return 1 + } + if L.currentFrame == nil || L.currentFrame.Fn.IsG || luaframe == baseframe { + return 1 + } + } else { + base := cf.Base + cf.Fn = callable + cf.Pc = 0 + cf.Base = RA + cf.LocalBase = RA + 1 + cf.ReturnBase = cf.ReturnBase + cf.NArgs = nargs + cf.NRet = cf.NRet + cf.TailCall++ + lbase := cf.LocalBase + if meta { + cf.NArgs++ + L.reg.Insert(lv, cf.LocalBase) + } + // +inline-call L.initCallFrame cf + // +inline-call L.reg.CopyRange base RA -1 reg.Top()-RA-1 + cf.Base = base + cf.LocalBase = base + (cf.LocalBase - lbase + 1) + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_RETURN + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + // +inline-call L.closeUpvalues lbase + nret := B - 1 + if B == 0 { + nret = reg.Top() - RA + } + n := cf.NRet + if cf.NRet == MultRet { + n = nret + } + + if L.Parent != nil && L.stack.Sp() == 1 { + // +inline-call copyReturnValues L reg.Top() RA n B + switchToParentThread(L, n, false, true) + return 1 + } + islast := baseframe == L.stack.Pop() || L.stack.IsEmpty() + // +inline-call copyReturnValues L cf.ReturnBase RA n B + L.currentFrame = L.stack.Last() + if islast || L.currentFrame == nil || L.currentFrame.Fn.IsG { + return 1 + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_FORLOOP + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + if init, ok1 := reg.Get(RA).(LNumber); ok1 { + if limit, ok2 := reg.Get(RA + 1).(LNumber); ok2 { + if step, ok3 := reg.Get(RA + 2).(LNumber); ok3 { + init += step + v := LNumber(init) + // +inline-call reg.SetNumber RA v + if (step > 0 && init <= limit) || (step <= 0 && init >= limit) { + Sbx := int(inst&0x3ffff) - opMaxArgSbx //GETSBX + cf.Pc += Sbx + // +inline-call reg.SetNumber RA+3 v + } else { + // +inline-call reg.SetTop RA+1 + } + } else { + L.RaiseError("for statement step must be a number") + } + } else { + L.RaiseError("for statement limit must be a number") + } + } else { + L.RaiseError("for statement init must be a number") + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_FORPREP + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Sbx := int(inst&0x3ffff) - opMaxArgSbx //GETSBX + if init, ok1 := reg.Get(RA).(LNumber); ok1 { + if step, ok2 := reg.Get(RA + 2).(LNumber); ok2 { + // +inline-call reg.SetNumber RA LNumber(init-step) + } else { + L.RaiseError("for statement step must be a number") + } + } else { + L.RaiseError("for statement init must be a number") + } + cf.Pc += Sbx + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_TFORLOOP + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + C := int(inst>>9) & 0x1ff //GETC + nret := C + // +inline-call reg.SetTop RA+3+2 + // +inline-call reg.Set RA+3+2 reg.Get(RA+2) + // +inline-call reg.Set RA+3+1 reg.Get(RA+1) + // +inline-call reg.Set RA+3 reg.Get(RA) + L.callR(2, nret, RA+3) + if value := reg.Get(RA + 3); value != LNil { + // +inline-call reg.Set RA+2 value + pc := cf.Fn.Proto.Code[cf.Pc] + cf.Pc += int(pc&0x3ffff) - opMaxArgSbx + } + cf.Pc++ + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETLIST + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + if C == 0 { + C = int(cf.Fn.Proto.Code[cf.Pc]) + cf.Pc++ + } + offset := (C - 1) * FieldsPerFlush + table := reg.Get(RA).(*LTable) + nelem := B + if B == 0 { + nelem = reg.Top() - RA - 1 + } + for i := 1; i <= nelem; i++ { + table.RawSetInt(offset+i, reg.Get(RA+i)) + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_CLOSE + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + // +inline-call L.closeUpvalues RA + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_CLOSURE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Bx := int(inst & 0x3ffff) //GETBX + proto := cf.Fn.Proto.FunctionPrototypes[Bx] + closure := newLFunctionL(proto, cf.Fn.Env, int(proto.NumUpvalues)) + // +inline-call reg.Set RA closure + for i := 0; i < int(proto.NumUpvalues); i++ { + inst = cf.Fn.Proto.Code[cf.Pc] + cf.Pc++ + B := opGetArgB(inst) + switch opGetOpCode(inst) { + case OP_MOVE: + closure.Upvalues[i] = L.findUpvalue(lbase + B) + case OP_GETUPVAL: + closure.Upvalues[i] = cf.Fn.Upvalues[B] + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_VARARG + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + nparams := int(cf.Fn.Proto.NumParameters) + nvarargs := cf.NArgs - nparams + if nvarargs < 0 { + nvarargs = 0 + } + nwant := B - 1 + if B == 0 { + nwant = nvarargs + } + // +inline-call reg.CopyRange RA cf.Base+nparams+1 cf.LocalBase nwant + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_NOP + return 0 + }, + } +} + +func opArith(L *LState, inst uint32, baseframe *callFrame) int { //OP_ADD, OP_SUB, OP_MUL, OP_DIV, OP_MOD, OP_POW + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + opcode := int(inst >> 26) //GETOPCODE + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + lhs := L.rkValue(B) + rhs := L.rkValue(C) + v1, ok1 := lhs.(LNumber) + v2, ok2 := rhs.(LNumber) + if ok1 && ok2 { + v := numberArith(L, opcode, LNumber(v1), LNumber(v2)) + // +inline-call reg.SetNumber RA v + } else { + v := objectArith(L, opcode, lhs, rhs) + // +inline-call reg.Set RA v + } + return 0 +} + +func luaModulo(lhs, rhs LNumber) LNumber { + flhs := float64(lhs) + frhs := float64(rhs) + v := math.Mod(flhs, frhs) + if frhs > 0 && v < 0 || frhs < 0 && v > 0 { + v += frhs + } + return LNumber(v) +} + +func numberArith(L *LState, opcode int, lhs, rhs LNumber) LNumber { + switch opcode { + case OP_ADD: + return lhs + rhs + case OP_SUB: + return lhs - rhs + case OP_MUL: + return lhs * rhs + case OP_DIV: + return lhs / rhs + case OP_MOD: + return luaModulo(lhs, rhs) + case OP_POW: + flhs := float64(lhs) + frhs := float64(rhs) + return LNumber(math.Pow(flhs, frhs)) + } + panic("should not reach here") + return LNumber(0) +} + +func objectArith(L *LState, opcode int, lhs, rhs LValue) LValue { + event := "" + switch opcode { + case OP_ADD: + event = "__add" + case OP_SUB: + event = "__sub" + case OP_MUL: + event = "__mul" + case OP_DIV: + event = "__div" + case OP_MOD: + event = "__mod" + case OP_POW: + event = "__pow" + } + op := L.metaOp2(lhs, rhs, event) + if _, ok := op.(*LFunction); ok { + L.reg.Push(op) + L.reg.Push(lhs) + L.reg.Push(rhs) + L.Call(2, 1) + return L.reg.Pop() + } + if str, ok := lhs.(LString); ok { + if lnum, err := parseNumber(string(str)); err == nil { + lhs = lnum + } + } + if str, ok := rhs.(LString); ok { + if rnum, err := parseNumber(string(str)); err == nil { + rhs = rnum + } + } + if v1, ok1 := lhs.(LNumber); ok1 { + if v2, ok2 := rhs.(LNumber); ok2 { + return numberArith(L, opcode, LNumber(v1), LNumber(v2)) + } + } + L.RaiseError(fmt.Sprintf("cannot perform %v operation between %v and %v", + strings.TrimLeft(event, "_"), lhs.Type().String(), rhs.Type().String())) + + return LNil +} + +func stringConcat(L *LState, total, last int) LValue { + rhs := L.reg.Get(last) + total-- + for i := last - 1; total > 0; { + lhs := L.reg.Get(i) + if !(LVCanConvToString(lhs) && LVCanConvToString(rhs)) { + op := L.metaOp2(lhs, rhs, "__concat") + if op.Type() == LTFunction { + L.reg.Push(op) + L.reg.Push(lhs) + L.reg.Push(rhs) + L.Call(2, 1) + rhs = L.reg.Pop() + total-- + i-- + } else { + L.RaiseError("cannot perform concat operation between %v and %v", lhs.Type().String(), rhs.Type().String()) + return LNil + } + } else { + buf := make([]string, total+1) + buf[total] = LVAsString(rhs) + for total > 0 { + lhs = L.reg.Get(i) + if !LVCanConvToString(lhs) { + break + } + buf[total-1] = LVAsString(lhs) + i-- + total-- + } + rhs = LString(strings.Join(buf, "")) + } + } + return rhs +} + +func lessThan(L *LState, lhs, rhs LValue) bool { + // optimization for numbers + if v1, ok1 := lhs.(LNumber); ok1 { + if v2, ok2 := rhs.(LNumber); ok2 { + return v1 < v2 + } + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + } + if lhs.Type() != rhs.Type() { + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + return false + } + ret := false + switch lhs.Type() { + case LTString: + ret = strCmp(string(lhs.(LString)), string(rhs.(LString))) < 0 + default: + ret = objectRationalWithError(L, lhs, rhs, "__lt") + } + return ret +} + +func equals(L *LState, lhs, rhs LValue, raw bool) bool { + lt := lhs.Type() + if lt != rhs.Type() { + return false + } + + ret := false + switch lt { + case LTNil: + ret = true + case LTNumber: + v1, _ := lhs.(LNumber) + v2, _ := rhs.(LNumber) + ret = v1 == v2 + case LTBool: + ret = bool(lhs.(LBool)) == bool(rhs.(LBool)) + case LTString: + ret = string(lhs.(LString)) == string(rhs.(LString)) + case LTUserData, LTTable: + if lhs == rhs { + ret = true + } else if !raw { + switch objectRational(L, lhs, rhs, "__eq") { + case 1: + ret = true + default: + ret = false + } + } + default: + ret = lhs == rhs + } + return ret +} + +func objectRationalWithError(L *LState, lhs, rhs LValue, event string) bool { + switch objectRational(L, lhs, rhs, event) { + case 1: + return true + case 0: + return false + } + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + return false +} + +func objectRational(L *LState, lhs, rhs LValue, event string) int { + m1 := L.metaOp1(lhs, event) + m2 := L.metaOp1(rhs, event) + if m1.Type() == LTFunction && m1 == m2 { + L.reg.Push(m1) + L.reg.Push(lhs) + L.reg.Push(rhs) + L.Call(2, 1) + if LVAsBool(L.reg.Pop()) { + return 1 + } + return 0 + } + return -1 +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/alloc.go b/scraper-go/vendor/github.com/yuin/gopher-lua/alloc.go new file mode 100644 index 00000000..7a8cd63a --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/alloc.go @@ -0,0 +1,79 @@ +package lua + +import ( + "reflect" + "unsafe" +) + +// iface is an internal representation of the go-interface. +type iface struct { + itab unsafe.Pointer + word unsafe.Pointer +} + +const preloadLimit LNumber = 128 + +var _fv float64 +var _uv uintptr + +var preloads [int(preloadLimit)]LValue + +func init() { + for i := 0; i < int(preloadLimit); i++ { + preloads[i] = LNumber(i) + } +} + +// allocator is a fast bulk memory allocator for the LValue. +type allocator struct { + size int + fptrs []float64 + fheader *reflect.SliceHeader + + scratchValue LValue + scratchValueP *iface +} + +func newAllocator(size int) *allocator { + al := &allocator{ + size: size, + fptrs: make([]float64, 0, size), + fheader: nil, + } + al.fheader = (*reflect.SliceHeader)(unsafe.Pointer(&al.fptrs)) + al.scratchValue = LNumber(0) + al.scratchValueP = (*iface)(unsafe.Pointer(&al.scratchValue)) + + return al +} + +// LNumber2I takes a number value and returns an interface LValue representing the same number. +// Converting an LNumber to a LValue naively, by doing: +// `var val LValue = myLNumber` +// will result in an individual heap alloc of 8 bytes for the float value. LNumber2I amortizes the cost and memory +// overhead of these allocs by allocating blocks of floats instead. +// The downside of this is that all of the floats on a given block have to become eligible for gc before the block +// as a whole can be gc-ed. +func (al *allocator) LNumber2I(v LNumber) LValue { + // first check for shared preloaded numbers + if v >= 0 && v < preloadLimit && float64(v) == float64(int64(v)) { + return preloads[int(v)] + } + + // check if we need a new alloc page + if cap(al.fptrs) == len(al.fptrs) { + al.fptrs = make([]float64, 0, al.size) + al.fheader = (*reflect.SliceHeader)(unsafe.Pointer(&al.fptrs)) + } + + // alloc a new float, and store our value into it + al.fptrs = append(al.fptrs, float64(v)) + fptr := &al.fptrs[len(al.fptrs)-1] + + // hack our scratch LValue to point to our allocated value + // this scratch lvalue is copied when this function returns meaning the scratch value can be reused + // on the next call + al.scratchValueP.word = unsafe.Pointer(fptr) + + return al.scratchValue +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/ast/ast.go b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/ast.go new file mode 100644 index 00000000..f337a294 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/ast.go @@ -0,0 +1,29 @@ +package ast + +type PositionHolder interface { + Line() int + SetLine(int) + LastLine() int + SetLastLine(int) +} + +type Node struct { + line int + lastline int +} + +func (self *Node) Line() int { + return self.line +} + +func (self *Node) SetLine(line int) { + self.line = line +} + +func (self *Node) LastLine() int { + return self.lastline +} + +func (self *Node) SetLastLine(line int) { + self.lastline = line +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/ast/expr.go b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/expr.go new file mode 100644 index 00000000..388852ba --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/expr.go @@ -0,0 +1,138 @@ +package ast + +type Expr interface { + PositionHolder + exprMarker() +} + +type ExprBase struct { + Node +} + +func (expr *ExprBase) exprMarker() {} + +/* ConstExprs {{{ */ + +type ConstExpr interface { + Expr + constExprMarker() +} + +type ConstExprBase struct { + ExprBase +} + +func (expr *ConstExprBase) constExprMarker() {} + +type TrueExpr struct { + ConstExprBase +} + +type FalseExpr struct { + ConstExprBase +} + +type NilExpr struct { + ConstExprBase +} + +type NumberExpr struct { + ConstExprBase + + Value string +} + +type StringExpr struct { + ConstExprBase + + Value string +} + +/* ConstExprs }}} */ + +type Comma3Expr struct { + ExprBase + AdjustRet bool +} + +type IdentExpr struct { + ExprBase + + Value string +} + +type AttrGetExpr struct { + ExprBase + + Object Expr + Key Expr +} + +type TableExpr struct { + ExprBase + + Fields []*Field +} + +type FuncCallExpr struct { + ExprBase + + Func Expr + Receiver Expr + Method string + Args []Expr + AdjustRet bool +} + +type LogicalOpExpr struct { + ExprBase + + Operator string + Lhs Expr + Rhs Expr +} + +type RelationalOpExpr struct { + ExprBase + + Operator string + Lhs Expr + Rhs Expr +} + +type StringConcatOpExpr struct { + ExprBase + + Lhs Expr + Rhs Expr +} + +type ArithmeticOpExpr struct { + ExprBase + + Operator string + Lhs Expr + Rhs Expr +} + +type UnaryMinusOpExpr struct { + ExprBase + Expr Expr +} + +type UnaryNotOpExpr struct { + ExprBase + Expr Expr +} + +type UnaryLenOpExpr struct { + ExprBase + Expr Expr +} + +type FunctionExpr struct { + ExprBase + + ParList *ParList + Stmts []Stmt +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/ast/misc.go b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/misc.go new file mode 100644 index 00000000..d811c042 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/misc.go @@ -0,0 +1,17 @@ +package ast + +type Field struct { + Key Expr + Value Expr +} + +type ParList struct { + HasVargs bool + Names []string +} + +type FuncName struct { + Func Expr + Receiver Expr + Method string +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/ast/stmt.go b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/stmt.go new file mode 100644 index 00000000..e24880b6 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/stmt.go @@ -0,0 +1,107 @@ +package ast + +type Stmt interface { + PositionHolder + stmtMarker() +} + +type StmtBase struct { + Node +} + +func (stmt *StmtBase) stmtMarker() {} + +type AssignStmt struct { + StmtBase + + Lhs []Expr + Rhs []Expr +} + +type LocalAssignStmt struct { + StmtBase + + Names []string + Exprs []Expr +} + +type FuncCallStmt struct { + StmtBase + + Expr Expr +} + +type DoBlockStmt struct { + StmtBase + + Stmts []Stmt +} + +type WhileStmt struct { + StmtBase + + Condition Expr + Stmts []Stmt +} + +type RepeatStmt struct { + StmtBase + + Condition Expr + Stmts []Stmt +} + +type IfStmt struct { + StmtBase + + Condition Expr + Then []Stmt + Else []Stmt +} + +type NumberForStmt struct { + StmtBase + + Name string + Init Expr + Limit Expr + Step Expr + Stmts []Stmt +} + +type GenericForStmt struct { + StmtBase + + Names []string + Exprs []Expr + Stmts []Stmt +} + +type FuncDefStmt struct { + StmtBase + + Name *FuncName + Func *FunctionExpr +} + +type ReturnStmt struct { + StmtBase + + Exprs []Expr +} + +type BreakStmt struct { + StmtBase +} + +type LabelStmt struct { + StmtBase + + Name string +} + +type GotoStmt struct { + StmtBase + + Label string +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/ast/token.go b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/token.go new file mode 100644 index 00000000..820467c9 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/ast/token.go @@ -0,0 +1,22 @@ +package ast + +import ( + "fmt" +) + +type Position struct { + Source string + Line int + Column int +} + +type Token struct { + Type int + Name string + Str string + Pos Position +} + +func (self *Token) String() string { + return fmt.Sprintf("", self.Name, self.Str) +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/auxlib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/auxlib.go new file mode 100644 index 00000000..a022bdd8 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/auxlib.go @@ -0,0 +1,465 @@ +package lua + +import ( + "bufio" + "fmt" + "io" + "os" + "strings" +) + +/* checkType {{{ */ + +func (ls *LState) CheckAny(n int) LValue { + if n > ls.GetTop() { + ls.ArgError(n, "value expected") + } + return ls.Get(n) +} + +func (ls *LState) CheckInt(n int) int { + v := ls.Get(n) + if intv, ok := v.(LNumber); ok { + return int(intv) + } + ls.TypeError(n, LTNumber) + return 0 +} + +func (ls *LState) CheckInt64(n int) int64 { + v := ls.Get(n) + if intv, ok := v.(LNumber); ok { + return int64(intv) + } + ls.TypeError(n, LTNumber) + return 0 +} + +func (ls *LState) CheckNumber(n int) LNumber { + v := ls.Get(n) + if lv, ok := v.(LNumber); ok { + return lv + } + if lv, ok := v.(LString); ok { + if num, err := parseNumber(string(lv)); err == nil { + return num + } + } + ls.TypeError(n, LTNumber) + return 0 +} + +func (ls *LState) CheckString(n int) string { + v := ls.Get(n) + if lv, ok := v.(LString); ok { + return string(lv) + } else if LVCanConvToString(v) { + return ls.ToString(n) + } + ls.TypeError(n, LTString) + return "" +} + +func (ls *LState) CheckBool(n int) bool { + v := ls.Get(n) + if lv, ok := v.(LBool); ok { + return bool(lv) + } + ls.TypeError(n, LTBool) + return false +} + +func (ls *LState) CheckTable(n int) *LTable { + v := ls.Get(n) + if lv, ok := v.(*LTable); ok { + return lv + } + ls.TypeError(n, LTTable) + return nil +} + +func (ls *LState) CheckFunction(n int) *LFunction { + v := ls.Get(n) + if lv, ok := v.(*LFunction); ok { + return lv + } + ls.TypeError(n, LTFunction) + return nil +} + +func (ls *LState) CheckUserData(n int) *LUserData { + v := ls.Get(n) + if lv, ok := v.(*LUserData); ok { + return lv + } + ls.TypeError(n, LTUserData) + return nil +} + +func (ls *LState) CheckThread(n int) *LState { + v := ls.Get(n) + if lv, ok := v.(*LState); ok { + return lv + } + ls.TypeError(n, LTThread) + return nil +} + +func (ls *LState) CheckType(n int, typ LValueType) { + v := ls.Get(n) + if v.Type() != typ { + ls.TypeError(n, typ) + } +} + +func (ls *LState) CheckTypes(n int, typs ...LValueType) { + vt := ls.Get(n).Type() + for _, typ := range typs { + if vt == typ { + return + } + } + buf := []string{} + for _, typ := range typs { + buf = append(buf, typ.String()) + } + ls.ArgError(n, strings.Join(buf, " or ")+" expected, got "+ls.Get(n).Type().String()) +} + +func (ls *LState) CheckOption(n int, options []string) int { + str := ls.CheckString(n) + for i, v := range options { + if v == str { + return i + } + } + ls.ArgError(n, fmt.Sprintf("invalid option: %s (must be one of %s)", str, strings.Join(options, ","))) + return 0 +} + +/* }}} */ + +/* optType {{{ */ + +func (ls *LState) OptInt(n int, d int) int { + v := ls.Get(n) + if v == LNil { + return d + } + if intv, ok := v.(LNumber); ok { + return int(intv) + } + ls.TypeError(n, LTNumber) + return 0 +} + +func (ls *LState) OptInt64(n int, d int64) int64 { + v := ls.Get(n) + if v == LNil { + return d + } + if intv, ok := v.(LNumber); ok { + return int64(intv) + } + ls.TypeError(n, LTNumber) + return 0 +} + +func (ls *LState) OptNumber(n int, d LNumber) LNumber { + v := ls.Get(n) + if v == LNil { + return d + } + if lv, ok := v.(LNumber); ok { + return lv + } + ls.TypeError(n, LTNumber) + return 0 +} + +func (ls *LState) OptString(n int, d string) string { + v := ls.Get(n) + if v == LNil { + return d + } + if lv, ok := v.(LString); ok { + return string(lv) + } + ls.TypeError(n, LTString) + return "" +} + +func (ls *LState) OptBool(n int, d bool) bool { + v := ls.Get(n) + if v == LNil { + return d + } + if lv, ok := v.(LBool); ok { + return bool(lv) + } + ls.TypeError(n, LTBool) + return false +} + +func (ls *LState) OptTable(n int, d *LTable) *LTable { + v := ls.Get(n) + if v == LNil { + return d + } + if lv, ok := v.(*LTable); ok { + return lv + } + ls.TypeError(n, LTTable) + return nil +} + +func (ls *LState) OptFunction(n int, d *LFunction) *LFunction { + v := ls.Get(n) + if v == LNil { + return d + } + if lv, ok := v.(*LFunction); ok { + return lv + } + ls.TypeError(n, LTFunction) + return nil +} + +func (ls *LState) OptUserData(n int, d *LUserData) *LUserData { + v := ls.Get(n) + if v == LNil { + return d + } + if lv, ok := v.(*LUserData); ok { + return lv + } + ls.TypeError(n, LTUserData) + return nil +} + +/* }}} */ + +/* error operations {{{ */ + +func (ls *LState) ArgError(n int, message string) { + ls.RaiseError("bad argument #%v to %v (%v)", n, ls.rawFrameFuncName(ls.currentFrame), message) +} + +func (ls *LState) TypeError(n int, typ LValueType) { + ls.RaiseError("bad argument #%v to %v (%v expected, got %v)", n, ls.rawFrameFuncName(ls.currentFrame), typ.String(), ls.Get(n).Type().String()) +} + +/* }}} */ + +/* debug operations {{{ */ + +func (ls *LState) Where(level int) string { + return ls.where(level, false) +} + +/* }}} */ + +/* table operations {{{ */ + +func (ls *LState) FindTable(obj *LTable, n string, size int) LValue { + names := strings.Split(n, ".") + curobj := obj + for _, name := range names { + if curobj.Type() != LTTable { + return LNil + } + nextobj := ls.RawGet(curobj, LString(name)) + if nextobj == LNil { + tb := ls.CreateTable(0, size) + ls.RawSet(curobj, LString(name), tb) + curobj = tb + } else if nextobj.Type() != LTTable { + return LNil + } else { + curobj = nextobj.(*LTable) + } + } + return curobj +} + +/* }}} */ + +/* register operations {{{ */ + +func (ls *LState) RegisterModule(name string, funcs map[string]LGFunction) LValue { + tb := ls.FindTable(ls.Get(RegistryIndex).(*LTable), "_LOADED", 1) + mod := ls.GetField(tb, name) + if mod.Type() != LTTable { + newmod := ls.FindTable(ls.Get(GlobalsIndex).(*LTable), name, len(funcs)) + if newmodtb, ok := newmod.(*LTable); !ok { + ls.RaiseError("name conflict for module(%v)", name) + } else { + for fname, fn := range funcs { + newmodtb.RawSetString(fname, ls.NewFunction(fn)) + } + ls.SetField(tb, name, newmodtb) + return newmodtb + } + } + return mod +} + +func (ls *LState) SetFuncs(tb *LTable, funcs map[string]LGFunction, upvalues ...LValue) *LTable { + for fname, fn := range funcs { + tb.RawSetString(fname, ls.NewClosure(fn, upvalues...)) + } + return tb +} + +/* }}} */ + +/* metatable operations {{{ */ + +func (ls *LState) NewTypeMetatable(typ string) *LTable { + regtable := ls.Get(RegistryIndex) + mt := ls.GetField(regtable, typ) + if tb, ok := mt.(*LTable); ok { + return tb + } + mtnew := ls.NewTable() + ls.SetField(regtable, typ, mtnew) + return mtnew +} + +func (ls *LState) GetMetaField(obj LValue, event string) LValue { + return ls.metaOp1(obj, event) +} + +func (ls *LState) GetTypeMetatable(typ string) LValue { + return ls.GetField(ls.Get(RegistryIndex), typ) +} + +func (ls *LState) CallMeta(obj LValue, event string) LValue { + op := ls.metaOp1(obj, event) + if op.Type() == LTFunction { + ls.reg.Push(op) + ls.reg.Push(obj) + ls.Call(1, 1) + return ls.reg.Pop() + } + return LNil +} + +/* }}} */ + +/* load and function call operations {{{ */ + +func (ls *LState) LoadFile(path string) (*LFunction, error) { + var file *os.File + var err error + if len(path) == 0 { + file = os.Stdin + } else { + file, err = os.Open(path) + defer file.Close() + if err != nil { + return nil, newApiErrorE(ApiErrorFile, err) + } + } + + reader := bufio.NewReader(file) + // get the first character. + c, err := reader.ReadByte() + if err != nil && err != io.EOF { + return nil, newApiErrorE(ApiErrorFile, err) + } + if c == byte('#') { + // Unix exec. file? + // skip first line + _, err, _ = readBufioLine(reader) + if err != nil { + return nil, newApiErrorE(ApiErrorFile, err) + } + } + + if err != io.EOF { + // if the file is not empty, + // unread the first character of the file or newline character(readBufioLine's last byte). + err = reader.UnreadByte() + if err != nil { + return nil, newApiErrorE(ApiErrorFile, err) + } + } + + return ls.Load(reader, path) +} + +func (ls *LState) LoadString(source string) (*LFunction, error) { + return ls.Load(strings.NewReader(source), "") +} + +func (ls *LState) DoFile(path string) error { + if fn, err := ls.LoadFile(path); err != nil { + return err + } else { + ls.Push(fn) + return ls.PCall(0, MultRet, nil) + } +} + +func (ls *LState) DoString(source string) error { + if fn, err := ls.LoadString(source); err != nil { + return err + } else { + ls.Push(fn) + return ls.PCall(0, MultRet, nil) + } +} + +/* }}} */ + +/* GopherLua original APIs {{{ */ + +// ToStringMeta returns string representation of given LValue. +// This method calls the `__tostring` meta method if defined. +func (ls *LState) ToStringMeta(lv LValue) LValue { + if fn, ok := ls.metaOp1(lv, "__tostring").(*LFunction); ok { + ls.Push(fn) + ls.Push(lv) + ls.Call(1, 1) + return ls.reg.Pop() + } else { + return LString(lv.String()) + } +} + +// Set a module loader to the package.preload table. +func (ls *LState) PreloadModule(name string, loader LGFunction) { + preload := ls.GetField(ls.GetField(ls.Get(EnvironIndex), "package"), "preload") + if _, ok := preload.(*LTable); !ok { + ls.RaiseError("package.preload must be a table") + } + ls.SetField(preload, name, ls.NewFunction(loader)) +} + +// Checks whether the given index is an LChannel and returns this channel. +func (ls *LState) CheckChannel(n int) chan LValue { + v := ls.Get(n) + if ch, ok := v.(LChannel); ok { + return (chan LValue)(ch) + } + ls.TypeError(n, LTChannel) + return nil +} + +// If the given index is a LChannel, returns this channel. If this argument is absent or is nil, returns ch. Otherwise, raises an error. +func (ls *LState) OptChannel(n int, ch chan LValue) chan LValue { + v := ls.Get(n) + if v == LNil { + return ch + } + if ch, ok := v.(LChannel); ok { + return (chan LValue)(ch) + } + ls.TypeError(n, LTChannel) + return nil +} + +/* }}} */ + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/baselib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/baselib.go new file mode 100644 index 00000000..aa2c08a9 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/baselib.go @@ -0,0 +1,597 @@ +package lua + +import ( + "fmt" + "io" + "os" + "runtime" + "strconv" + "strings" +) + +/* basic functions {{{ */ + +func OpenBase(L *LState) int { + global := L.Get(GlobalsIndex).(*LTable) + L.SetGlobal("_G", global) + L.SetGlobal("_VERSION", LString(LuaVersion)) + L.SetGlobal("_GOPHER_LUA_VERSION", LString(PackageName+" "+PackageVersion)) + basemod := L.RegisterModule("_G", baseFuncs) + global.RawSetString("ipairs", L.NewClosure(baseIpairs, L.NewFunction(ipairsaux))) + global.RawSetString("pairs", L.NewClosure(basePairs, L.NewFunction(pairsaux))) + L.Push(basemod) + return 1 +} + +var baseFuncs = map[string]LGFunction{ + "assert": baseAssert, + "collectgarbage": baseCollectGarbage, + "dofile": baseDoFile, + "error": baseError, + "getfenv": baseGetFEnv, + "getmetatable": baseGetMetatable, + "load": baseLoad, + "loadfile": baseLoadFile, + "loadstring": baseLoadString, + "next": baseNext, + "pcall": basePCall, + "print": basePrint, + "rawequal": baseRawEqual, + "rawget": baseRawGet, + "rawset": baseRawSet, + "select": baseSelect, + "_printregs": base_PrintRegs, + "setfenv": baseSetFEnv, + "setmetatable": baseSetMetatable, + "tonumber": baseToNumber, + "tostring": baseToString, + "type": baseType, + "unpack": baseUnpack, + "xpcall": baseXPCall, + // loadlib + "module": loModule, + "require": loRequire, + // hidden features + "newproxy": baseNewProxy, +} + +func baseAssert(L *LState) int { + if !L.ToBool(1) { + L.RaiseError(L.OptString(2, "assertion failed!")) + return 0 + } + return L.GetTop() +} + +func baseCollectGarbage(L *LState) int { + runtime.GC() + return 0 +} + +func baseDoFile(L *LState) int { + src := L.ToString(1) + top := L.GetTop() + fn, err := L.LoadFile(src) + if err != nil { + L.Push(LString(err.Error())) + L.Panic(L) + } + L.Push(fn) + L.Call(0, MultRet) + return L.GetTop() - top +} + +func baseError(L *LState) int { + obj := L.CheckAny(1) + level := L.OptInt(2, 1) + L.Error(obj, level) + return 0 +} + +func baseGetFEnv(L *LState) int { + var value LValue + if L.GetTop() == 0 { + value = LNumber(1) + } else { + value = L.Get(1) + } + + if fn, ok := value.(*LFunction); ok { + if !fn.IsG { + L.Push(fn.Env) + } else { + L.Push(L.G.Global) + } + return 1 + } + + if number, ok := value.(LNumber); ok { + level := int(float64(number)) + if level <= 0 { + L.Push(L.Env) + } else { + cf := L.currentFrame + for i := 0; i < level && cf != nil; i++ { + cf = cf.Parent + } + if cf == nil || cf.Fn.IsG { + L.Push(L.G.Global) + } else { + L.Push(cf.Fn.Env) + } + } + return 1 + } + + L.Push(L.G.Global) + return 1 +} + +func baseGetMetatable(L *LState) int { + L.Push(L.GetMetatable(L.CheckAny(1))) + return 1 +} + +func ipairsaux(L *LState) int { + tb := L.CheckTable(1) + i := L.CheckInt(2) + i++ + v := tb.RawGetInt(i) + if v == LNil { + return 0 + } else { + L.Pop(1) + L.Push(LNumber(i)) + L.Push(LNumber(i)) + L.Push(v) + return 2 + } +} + +func baseIpairs(L *LState) int { + tb := L.CheckTable(1) + L.Push(L.Get(UpvalueIndex(1))) + L.Push(tb) + L.Push(LNumber(0)) + return 3 +} + +func loadaux(L *LState, reader io.Reader, chunkname string) int { + if fn, err := L.Load(reader, chunkname); err != nil { + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 + } else { + L.Push(fn) + return 1 + } +} + +func baseLoad(L *LState) int { + fn := L.CheckFunction(1) + chunkname := L.OptString(2, "?") + top := L.GetTop() + buf := []string{} + for { + L.SetTop(top) + L.Push(fn) + L.Call(0, 1) + ret := L.reg.Pop() + if ret == LNil { + break + } else if LVCanConvToString(ret) { + str := ret.String() + if len(str) > 0 { + buf = append(buf, string(str)) + } else { + break + } + } else { + L.Push(LNil) + L.Push(LString("reader function must return a string")) + return 2 + } + } + return loadaux(L, strings.NewReader(strings.Join(buf, "")), chunkname) +} + +func baseLoadFile(L *LState) int { + var reader io.Reader + var chunkname string + var err error + if L.GetTop() < 1 { + reader = os.Stdin + chunkname = "" + } else { + chunkname = L.CheckString(1) + reader, err = os.Open(chunkname) + if err != nil { + L.Push(LNil) + L.Push(LString(fmt.Sprintf("can not open file: %v", chunkname))) + return 2 + } + defer reader.(*os.File).Close() + } + return loadaux(L, reader, chunkname) +} + +func baseLoadString(L *LState) int { + return loadaux(L, strings.NewReader(L.CheckString(1)), L.OptString(2, "")) +} + +func baseNext(L *LState) int { + tb := L.CheckTable(1) + index := LNil + if L.GetTop() >= 2 { + index = L.Get(2) + } + key, value := tb.Next(index) + if key == LNil { + L.Push(LNil) + return 1 + } + L.Push(key) + L.Push(value) + return 2 +} + +func pairsaux(L *LState) int { + tb := L.CheckTable(1) + key, value := tb.Next(L.Get(2)) + if key == LNil { + return 0 + } else { + L.Pop(1) + L.Push(key) + L.Push(key) + L.Push(value) + return 2 + } +} + +func basePairs(L *LState) int { + tb := L.CheckTable(1) + L.Push(L.Get(UpvalueIndex(1))) + L.Push(tb) + L.Push(LNil) + return 3 +} + +func basePCall(L *LState) int { + L.CheckAny(1) + v := L.Get(1) + if v.Type() != LTFunction && L.GetMetaField(v, "__call").Type() != LTFunction { + L.Push(LFalse) + L.Push(LString("attempt to call a " + v.Type().String() + " value")) + return 2 + } + nargs := L.GetTop() - 1 + if err := L.PCall(nargs, MultRet, nil); err != nil { + L.Push(LFalse) + if aerr, ok := err.(*ApiError); ok { + L.Push(aerr.Object) + } else { + L.Push(LString(err.Error())) + } + return 2 + } else { + L.Insert(LTrue, 1) + return L.GetTop() + } +} + +func basePrint(L *LState) int { + top := L.GetTop() + for i := 1; i <= top; i++ { + fmt.Print(L.ToStringMeta(L.Get(i)).String()) + if i != top { + fmt.Print("\t") + } + } + fmt.Println("") + return 0 +} + +func base_PrintRegs(L *LState) int { + L.printReg() + return 0 +} + +func baseRawEqual(L *LState) int { + if L.CheckAny(1) == L.CheckAny(2) { + L.Push(LTrue) + } else { + L.Push(LFalse) + } + return 1 +} + +func baseRawGet(L *LState) int { + L.Push(L.RawGet(L.CheckTable(1), L.CheckAny(2))) + return 1 +} + +func baseRawSet(L *LState) int { + L.RawSet(L.CheckTable(1), L.CheckAny(2), L.CheckAny(3)) + return 0 +} + +func baseSelect(L *LState) int { + L.CheckTypes(1, LTNumber, LTString) + switch lv := L.Get(1).(type) { + case LNumber: + idx := int(lv) + num := L.GetTop() + if idx < 0 { + idx = num + idx + } else if idx > num { + idx = num + } + if 1 > idx { + L.ArgError(1, "index out of range") + } + return num - idx + case LString: + if string(lv) != "#" { + L.ArgError(1, "invalid string '"+string(lv)+"'") + } + L.Push(LNumber(L.GetTop() - 1)) + return 1 + } + return 0 +} + +func baseSetFEnv(L *LState) int { + var value LValue + if L.GetTop() == 0 { + value = LNumber(1) + } else { + value = L.Get(1) + } + env := L.CheckTable(2) + + if fn, ok := value.(*LFunction); ok { + if fn.IsG { + L.RaiseError("cannot change the environment of given object") + } else { + fn.Env = env + L.Push(fn) + return 1 + } + } + + if number, ok := value.(LNumber); ok { + level := int(float64(number)) + if level <= 0 { + L.Env = env + return 0 + } + + cf := L.currentFrame + for i := 0; i < level && cf != nil; i++ { + cf = cf.Parent + } + if cf == nil || cf.Fn.IsG { + L.RaiseError("cannot change the environment of given object") + } else { + cf.Fn.Env = env + L.Push(cf.Fn) + return 1 + } + } + + L.RaiseError("cannot change the environment of given object") + return 0 +} + +func baseSetMetatable(L *LState) int { + L.CheckTypes(2, LTNil, LTTable) + obj := L.Get(1) + if obj == LNil { + L.RaiseError("cannot set metatable to a nil object.") + } + mt := L.Get(2) + if m := L.metatable(obj, true); m != LNil { + if tb, ok := m.(*LTable); ok && tb.RawGetString("__metatable") != LNil { + L.RaiseError("cannot change a protected metatable") + } + } + L.SetMetatable(obj, mt) + L.SetTop(1) + return 1 +} + +func baseToNumber(L *LState) int { + base := L.OptInt(2, 10) + noBase := L.Get(2) == LNil + + switch lv := L.CheckAny(1).(type) { + case LNumber: + L.Push(lv) + case LString: + str := strings.Trim(string(lv), " \n\t") + if strings.Index(str, ".") > -1 { + if v, err := strconv.ParseFloat(str, LNumberBit); err != nil { + L.Push(LNil) + } else { + L.Push(LNumber(v)) + } + } else { + if noBase && strings.HasPrefix(strings.ToLower(str), "0x") { + base, str = 16, str[2:] // Hex number + } + if v, err := strconv.ParseInt(str, base, LNumberBit); err != nil { + L.Push(LNil) + } else { + L.Push(LNumber(v)) + } + } + default: + L.Push(LNil) + } + return 1 +} + +func baseToString(L *LState) int { + v1 := L.CheckAny(1) + L.Push(L.ToStringMeta(v1)) + return 1 +} + +func baseType(L *LState) int { + L.Push(LString(L.CheckAny(1).Type().String())) + return 1 +} + +func baseUnpack(L *LState) int { + tb := L.CheckTable(1) + start := L.OptInt(2, 1) + end := L.OptInt(3, tb.Len()) + for i := start; i <= end; i++ { + L.Push(tb.RawGetInt(i)) + } + ret := end - start + 1 + if ret < 0 { + return 0 + } + return ret +} + +func baseXPCall(L *LState) int { + fn := L.CheckFunction(1) + errfunc := L.CheckFunction(2) + + top := L.GetTop() + L.Push(fn) + if err := L.PCall(0, MultRet, errfunc); err != nil { + L.Push(LFalse) + if aerr, ok := err.(*ApiError); ok { + L.Push(aerr.Object) + } else { + L.Push(LString(err.Error())) + } + return 2 + } else { + L.Insert(LTrue, top+1) + return L.GetTop() - top + } +} + +/* }}} */ + +/* load lib {{{ */ + +func loModule(L *LState) int { + name := L.CheckString(1) + loaded := L.GetField(L.Get(RegistryIndex), "_LOADED") + tb := L.GetField(loaded, name) + if _, ok := tb.(*LTable); !ok { + tb = L.FindTable(L.Get(GlobalsIndex).(*LTable), name, 1) + if tb == LNil { + L.RaiseError("name conflict for module: %v", name) + } + L.SetField(loaded, name, tb) + } + if L.GetField(tb, "_NAME") == LNil { + L.SetField(tb, "_M", tb) + L.SetField(tb, "_NAME", LString(name)) + names := strings.Split(name, ".") + pname := "" + if len(names) > 1 { + pname = strings.Join(names[:len(names)-1], ".") + "." + } + L.SetField(tb, "_PACKAGE", LString(pname)) + } + + caller := L.currentFrame.Parent + if caller == nil { + L.RaiseError("no calling stack.") + } else if caller.Fn.IsG { + L.RaiseError("module() can not be called from GFunctions.") + } + L.SetFEnv(caller.Fn, tb) + + top := L.GetTop() + for i := 2; i <= top; i++ { + L.Push(L.Get(i)) + L.Push(tb) + L.Call(1, 0) + } + L.Push(tb) + return 1 +} + +var loopdetection = &LUserData{} + +func loRequire(L *LState) int { + name := L.CheckString(1) + loaded := L.GetField(L.Get(RegistryIndex), "_LOADED") + lv := L.GetField(loaded, name) + if LVAsBool(lv) { + if lv == loopdetection { + L.RaiseError("loop or previous error loading module: %s", name) + } + L.Push(lv) + return 1 + } + loaders, ok := L.GetField(L.Get(RegistryIndex), "_LOADERS").(*LTable) + if !ok { + L.RaiseError("package.loaders must be a table") + } + messages := []string{} + var modasfunc LValue + for i := 1; ; i++ { + loader := L.RawGetInt(loaders, i) + if loader == LNil { + L.RaiseError("module %s not found:\n\t%s, ", name, strings.Join(messages, "\n\t")) + } + L.Push(loader) + L.Push(LString(name)) + L.Call(1, 1) + ret := L.reg.Pop() + switch retv := ret.(type) { + case *LFunction: + modasfunc = retv + goto loopbreak + case LString: + messages = append(messages, string(retv)) + } + } +loopbreak: + L.SetField(loaded, name, loopdetection) + L.Push(modasfunc) + L.Push(LString(name)) + L.Call(1, 1) + ret := L.reg.Pop() + modv := L.GetField(loaded, name) + if ret != LNil && modv == loopdetection { + L.SetField(loaded, name, ret) + L.Push(ret) + } else if modv == loopdetection { + L.SetField(loaded, name, LTrue) + L.Push(LTrue) + } else { + L.Push(modv) + } + return 1 +} + +/* }}} */ + +/* hidden features {{{ */ + +func baseNewProxy(L *LState) int { + ud := L.NewUserData() + L.SetTop(1) + if L.Get(1) == LTrue { + L.SetMetatable(ud, L.NewTable()) + } else if d, ok := L.Get(1).(*LUserData); ok { + L.SetMetatable(ud, L.GetMetatable(d)) + } + L.Push(ud) + return 1 +} + +/* }}} */ + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/channellib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/channellib.go new file mode 100644 index 00000000..a92bf72c --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/channellib.go @@ -0,0 +1,184 @@ +package lua + +import ( + "reflect" +) + +func checkChannel(L *LState, idx int) reflect.Value { + ch := L.CheckChannel(idx) + return reflect.ValueOf(ch) +} + +func checkGoroutineSafe(L *LState, idx int) LValue { + v := L.CheckAny(2) + if !isGoroutineSafe(v) { + L.ArgError(2, "can not send a function, userdata, thread or table that has a metatable") + } + return v +} + +func OpenChannel(L *LState) int { + var mod LValue + //_, ok := L.G.builtinMts[int(LTChannel)] + // if !ok { + mod = L.RegisterModule(ChannelLibName, channelFuncs) + mt := L.SetFuncs(L.NewTable(), channelMethods) + mt.RawSetString("__index", mt) + L.G.builtinMts[int(LTChannel)] = mt + // } + L.Push(mod) + return 1 +} + +var channelFuncs = map[string]LGFunction{ + "make": channelMake, + "select": channelSelect, +} + +func channelMake(L *LState) int { + buffer := L.OptInt(1, 0) + L.Push(LChannel(make(chan LValue, buffer))) + return 1 +} + +func channelSelect(L *LState) int { + //TODO check case table size + cases := make([]reflect.SelectCase, L.GetTop()) + top := L.GetTop() + for i := 0; i < top; i++ { + cas := reflect.SelectCase{ + Dir: reflect.SelectSend, + Chan: reflect.ValueOf(nil), + Send: reflect.ValueOf(nil), + } + tbl := L.CheckTable(i + 1) + dir, ok1 := tbl.RawGetInt(1).(LString) + if !ok1 { + L.ArgError(i+1, "invalid select case") + } + switch string(dir) { + case "<-|": + ch, ok := tbl.RawGetInt(2).(LChannel) + if !ok { + L.ArgError(i+1, "invalid select case") + } + cas.Chan = reflect.ValueOf((chan LValue)(ch)) + v := tbl.RawGetInt(3) + if !isGoroutineSafe(v) { + L.ArgError(i+1, "can not send a function, userdata, thread or table that has a metatable") + } + cas.Send = reflect.ValueOf(v) + case "|<-": + ch, ok := tbl.RawGetInt(2).(LChannel) + if !ok { + L.ArgError(i+1, "invalid select case") + } + cas.Chan = reflect.ValueOf((chan LValue)(ch)) + cas.Dir = reflect.SelectRecv + case "default": + cas.Dir = reflect.SelectDefault + default: + L.ArgError(i+1, "invalid channel direction:"+string(dir)) + } + cases[i] = cas + } + + if L.ctx != nil { + cases = append(cases, reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(L.ctx.Done()), + Send: reflect.ValueOf(nil), + }) + } + + pos, recv, rok := reflect.Select(cases) + + if L.ctx != nil && pos == L.GetTop() { + return 0 + } + + lv := LNil + if recv.Kind() != 0 { + lv, _ = recv.Interface().(LValue) + if lv == nil { + lv = LNil + } + } + tbl := L.Get(pos + 1).(*LTable) + last := tbl.RawGetInt(tbl.Len()) + if last.Type() == LTFunction { + L.Push(last) + switch cases[pos].Dir { + case reflect.SelectRecv: + if rok { + L.Push(LTrue) + } else { + L.Push(LFalse) + } + L.Push(lv) + L.Call(2, 0) + case reflect.SelectSend: + L.Push(tbl.RawGetInt(3)) + L.Call(1, 0) + case reflect.SelectDefault: + L.Call(0, 0) + } + } + L.Push(LNumber(pos + 1)) + L.Push(lv) + if rok { + L.Push(LTrue) + } else { + L.Push(LFalse) + } + return 3 +} + +var channelMethods = map[string]LGFunction{ + "receive": channelReceive, + "send": channelSend, + "close": channelClose, +} + +func channelReceive(L *LState) int { + rch := checkChannel(L, 1) + var v reflect.Value + var ok bool + if L.ctx != nil { + cases := []reflect.SelectCase{{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(L.ctx.Done()), + Send: reflect.ValueOf(nil), + }, { + Dir: reflect.SelectRecv, + Chan: rch, + Send: reflect.ValueOf(nil), + }} + _, v, ok = reflect.Select(cases) + } else { + v, ok = rch.Recv() + } + if ok { + L.Push(LTrue) + L.Push(v.Interface().(LValue)) + } else { + L.Push(LFalse) + L.Push(LNil) + } + return 2 +} + +func channelSend(L *LState) int { + rch := checkChannel(L, 1) + v := checkGoroutineSafe(L, 2) + rch.Send(reflect.ValueOf(v)) + return 0 +} + +func channelClose(L *LState) int { + rch := checkChannel(L, 1) + rch.Close() + return 0 +} + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/compile.go b/scraper-go/vendor/github.com/yuin/gopher-lua/compile.go new file mode 100644 index 00000000..c3736777 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/compile.go @@ -0,0 +1,1869 @@ +package lua + +import ( + "fmt" + "math" + "reflect" + + "github.com/yuin/gopher-lua/ast" +) + +/* internal constants & structs {{{ */ + +const maxRegisters = 200 + +type expContextType int + +const ( + ecGlobal expContextType = iota + ecUpvalue + ecLocal + ecTable + ecVararg + ecMethod + ecNone +) + +const regNotDefined = opMaxArgsA + 1 +const labelNoJump = 0 + +type expcontext struct { + ctype expContextType + reg int + // varargopt >= 0: wants varargopt+1 results, i.e a = func() + // varargopt = -1: ignore results i.e func() + // varargopt = -2: receive all results i.e a = {func()} + varargopt int +} + +type assigncontext struct { + ec *expcontext + keyrk int + valuerk int + keyks bool + needmove bool +} + +type lblabels struct { + t int + f int + e int + b bool +} + +type constLValueExpr struct { + ast.ExprBase + + Value LValue +} + +// }}} + +/* utilities {{{ */ +var _ecnone0 = &expcontext{ecNone, regNotDefined, 0} +var _ecnonem1 = &expcontext{ecNone, regNotDefined, -1} +var _ecnonem2 = &expcontext{ecNone, regNotDefined, -2} +var ecfuncdef = &expcontext{ecMethod, regNotDefined, 0} + +func ecupdate(ec *expcontext, ctype expContextType, reg, varargopt int) { + if ec == _ecnone0 || ec == _ecnonem1 || ec == _ecnonem2 { + panic("can not update ec cache") + } + ec.ctype = ctype + ec.reg = reg + ec.varargopt = varargopt +} + +func ecnone(varargopt int) *expcontext { + switch varargopt { + case 0: + return _ecnone0 + case -1: + return _ecnonem1 + case -2: + return _ecnonem2 + } + return &expcontext{ecNone, regNotDefined, varargopt} +} + +func shouldmove(ec *expcontext, reg int) bool { + return ec.ctype == ecLocal && ec.reg != regNotDefined && ec.reg != reg +} + +func sline(pos ast.PositionHolder) int { + return pos.Line() +} + +func eline(pos ast.PositionHolder) int { + line := pos.LastLine() + if line == 0 { + return pos.Line() + } + return line +} + +func savereg(ec *expcontext, reg int) int { + if ec.ctype != ecLocal || ec.reg == regNotDefined { + return reg + } + return ec.reg +} + +func raiseCompileError(context *funcContext, line int, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + panic(&CompileError{context: context, Line: line, Message: msg}) +} + +func isVarArgReturnExpr(expr ast.Expr) bool { + switch ex := expr.(type) { + case *ast.FuncCallExpr: + return !ex.AdjustRet + case *ast.Comma3Expr: + return !ex.AdjustRet + } + return false +} + +func lnumberValue(expr ast.Expr) (LNumber, bool) { + if ex, ok := expr.(*ast.NumberExpr); ok { + lv, err := parseNumber(ex.Value) + if err != nil { + lv = LNumber(math.NaN()) + } + return lv, true + } else if ex, ok := expr.(*constLValueExpr); ok { + return ex.Value.(LNumber), true + } + return 0, false +} + +/* utilities }}} */ + +type gotoLabelDesc struct { // {{{ + Id int + Name string + Pc int + Line int + NumActiveLocalVars int +} + +func newLabelDesc(id int, name string, pc, line, n int) *gotoLabelDesc { + return &gotoLabelDesc{ + Id: id, + Name: name, + Pc: pc, + Line: line, + NumActiveLocalVars: n, + } +} + +func (l *gotoLabelDesc) SetNumActiveLocalVars(n int) { + l.NumActiveLocalVars = n +} // }}} + +type CompileError struct { // {{{ + context *funcContext + Line int + Message string +} + +func (e *CompileError) Error() string { + return fmt.Sprintf("compile error near line(%v) %v: %v", e.Line, e.context.Proto.SourceName, e.Message) +} // }}} + +type codeStore struct { // {{{ + codes []uint32 + lines []int + pc int +} + +func (cd *codeStore) Add(inst uint32, line int) { + if l := len(cd.codes); l <= 0 || cd.pc == l { + cd.codes = append(cd.codes, inst) + cd.lines = append(cd.lines, line) + } else { + cd.codes[cd.pc] = inst + cd.lines[cd.pc] = line + } + cd.pc++ +} + +func (cd *codeStore) AddABC(op int, a int, b int, c int, line int) { + cd.Add(opCreateABC(op, a, b, c), line) +} + +func (cd *codeStore) AddABx(op int, a int, bx int, line int) { + cd.Add(opCreateABx(op, a, bx), line) +} + +func (cd *codeStore) AddASbx(op int, a int, sbx int, line int) { + cd.Add(opCreateASbx(op, a, sbx), line) +} + +func (cd *codeStore) PropagateKMV(top int, save *int, reg *int, inc int) { + lastinst := cd.Last() + if opGetArgA(lastinst) >= top { + switch opGetOpCode(lastinst) { + case OP_LOADK: + cindex := opGetArgBx(lastinst) + if cindex <= opMaxIndexRk { + cd.Pop() + *save = opRkAsk(cindex) + return + } + case OP_MOVE: + cd.Pop() + *save = opGetArgB(lastinst) + return + } + } + *save = *reg + *reg = *reg + inc +} + +func (cd *codeStore) PropagateMV(top int, save *int, reg *int, inc int) { + lastinst := cd.Last() + if opGetArgA(lastinst) >= top { + switch opGetOpCode(lastinst) { + case OP_MOVE: + cd.Pop() + *save = opGetArgB(lastinst) + return + } + } + *save = *reg + *reg = *reg + inc +} + +func (cd *codeStore) AddLoadNil(a, b, line int) { + last := cd.Last() + if opGetOpCode(last) == OP_LOADNIL && (opGetArgA(last)+opGetArgB(last)) == a { + cd.SetB(cd.LastPC(), b) + } else { + cd.AddABC(OP_LOADNIL, a, b, 0, line) + } +} + +func (cd *codeStore) SetOpCode(pc int, v int) { + opSetOpCode(&cd.codes[pc], v) +} + +func (cd *codeStore) SetA(pc int, v int) { + opSetArgA(&cd.codes[pc], v) +} + +func (cd *codeStore) SetB(pc int, v int) { + opSetArgB(&cd.codes[pc], v) +} + +func (cd *codeStore) SetC(pc int, v int) { + opSetArgC(&cd.codes[pc], v) +} + +func (cd *codeStore) SetBx(pc int, v int) { + opSetArgBx(&cd.codes[pc], v) +} + +func (cd *codeStore) SetSbx(pc int, v int) { + opSetArgSbx(&cd.codes[pc], v) +} + +func (cd *codeStore) At(pc int) uint32 { + return cd.codes[pc] +} + +func (cd *codeStore) List() []uint32 { + return cd.codes[:cd.pc] +} + +func (cd *codeStore) PosList() []int { + return cd.lines[:cd.pc] +} + +func (cd *codeStore) LastPC() int { + return cd.pc - 1 +} + +func (cd *codeStore) Last() uint32 { + if cd.pc == 0 { + return opInvalidInstruction + } + return cd.codes[cd.pc-1] +} + +func (cd *codeStore) Pop() { + cd.pc-- +} /* }}} Code */ + +/* {{{ VarNamePool */ + +type varNamePoolValue struct { + Index int + Name string +} + +type varNamePool struct { + names []string + offset int +} + +func newVarNamePool(offset int) *varNamePool { + return &varNamePool{make([]string, 0, 16), offset} +} + +func (vp *varNamePool) Names() []string { + return vp.names +} + +func (vp *varNamePool) List() []varNamePoolValue { + result := make([]varNamePoolValue, len(vp.names), len(vp.names)) + for i, name := range vp.names { + result[i].Index = i + vp.offset + result[i].Name = name + } + return result +} + +func (vp *varNamePool) LastIndex() int { + return vp.offset + len(vp.names) +} + +func (vp *varNamePool) Find(name string) int { + for i := len(vp.names) - 1; i >= 0; i-- { + if vp.names[i] == name { + return i + vp.offset + } + } + return -1 +} + +func (vp *varNamePool) RegisterUnique(name string) int { + index := vp.Find(name) + if index < 0 { + return vp.Register(name) + } + return index +} + +func (vp *varNamePool) Register(name string) int { + vp.names = append(vp.names, name) + return len(vp.names) - 1 + vp.offset +} + +/* }}} VarNamePool */ + +/* FuncContext {{{ */ + +type codeBlock struct { + LocalVars *varNamePool + BreakLabel int + Parent *codeBlock + RefUpvalue bool + LineStart int + LastLine int + labels map[string]*gotoLabelDesc + firstGotoIndex int +} + +func newCodeBlock(localvars *varNamePool, blabel int, parent *codeBlock, pos ast.PositionHolder, firstGotoIndex int) *codeBlock { + bl := &codeBlock{localvars, blabel, parent, false, 0, 0, map[string]*gotoLabelDesc{}, firstGotoIndex} + if pos != nil { + bl.LineStart = pos.Line() + bl.LastLine = pos.LastLine() + } + return bl +} + +func (b *codeBlock) AddLabel(label *gotoLabelDesc) *gotoLabelDesc { + if old, ok := b.labels[label.Name]; ok { + return old + } + b.labels[label.Name] = label + return nil +} + +func (b *codeBlock) GetLabel(label string) *gotoLabelDesc { + if v, ok := b.labels[label]; ok { + return v + } + return nil +} + +func (b *codeBlock) LocalVarsCount() int { + count := 0 + for block := b; block != nil; block = block.Parent { + count += len(block.LocalVars.Names()) + } + return count +} + +type funcContext struct { + Proto *FunctionProto + Code *codeStore + Parent *funcContext + Upvalues *varNamePool + Block *codeBlock + Blocks []*codeBlock + regTop int + labelId int + labelPc map[int]int + gotosCount int + unresolvedGotos map[int]*gotoLabelDesc +} + +func newFuncContext(sourcename string, parent *funcContext) *funcContext { + fc := &funcContext{ + Proto: newFunctionProto(sourcename), + Code: &codeStore{make([]uint32, 0, 1024), make([]int, 0, 1024), 0}, + Parent: parent, + Upvalues: newVarNamePool(0), + Block: newCodeBlock(newVarNamePool(0), labelNoJump, nil, nil, 0), + regTop: 0, + labelId: 1, + labelPc: map[int]int{}, + gotosCount: 0, + unresolvedGotos: map[int]*gotoLabelDesc{}, + } + fc.Blocks = []*codeBlock{fc.Block} + return fc +} + +func (fc *funcContext) CheckUnresolvedGoto() { + for i := fc.Block.firstGotoIndex; i < fc.gotosCount; i++ { + gotoLabel, ok := fc.unresolvedGotos[i] + if !ok { + continue + } + raiseCompileError(fc, fc.Proto.LastLineDefined, "no visible label '%s' for at line %d", gotoLabel.Name, gotoLabel.Line) + } +} + +func (fc *funcContext) AddUnresolvedGoto(label *gotoLabelDesc) { + fc.unresolvedGotos[fc.gotosCount] = label + fc.gotosCount++ +} + +func (fc *funcContext) AddNamedLabel(label *gotoLabelDesc) { + if old := fc.Block.AddLabel(label); old != nil { + raiseCompileError(fc, label.Line+1, "label '%s' already defined on line %d", label.Name, old.Line) + } + fc.SetLabelPc(label.Id, label.Pc) +} + +func (fc *funcContext) GetNamedLabel(name string) *gotoLabelDesc { + return fc.Block.GetLabel(name) +} + +func (fc *funcContext) ResolveGoto(from, to *gotoLabelDesc, index int) { + if from.NumActiveLocalVars < to.NumActiveLocalVars { + varName := fc.Block.LocalVars.Names()[len(fc.Block.LocalVars.Names())-1] + raiseCompileError(fc, to.Line+1, " at line %d jumps into the scope of local '%s'", to.Name, from.Line, varName) + } + fc.Code.SetSbx(from.Pc, to.Id) + delete(fc.unresolvedGotos, index) +} + +func (fc *funcContext) FindLabel(block *codeBlock, gotoLabel *gotoLabelDesc, i int) bool { + target := block.GetLabel(gotoLabel.Name) + if target != nil { + if gotoLabel.NumActiveLocalVars > target.NumActiveLocalVars && block.RefUpvalue { + fc.Code.SetA(gotoLabel.Pc-1, target.NumActiveLocalVars) + } + fc.ResolveGoto(gotoLabel, target, i) + return true + } + return false +} + +func (fc *funcContext) ResolveCurrentBlockGotosWithParentBlock() { + blockActiveLocalVars := fc.Block.Parent.LocalVarsCount() + for i := fc.Block.firstGotoIndex; i < fc.gotosCount; i++ { + gotoLabel, ok := fc.unresolvedGotos[i] + if !ok { + continue + } + if gotoLabel.NumActiveLocalVars > blockActiveLocalVars { + if fc.Block.RefUpvalue { + fc.Code.SetA(gotoLabel.Pc-1, blockActiveLocalVars) + } + gotoLabel.SetNumActiveLocalVars(blockActiveLocalVars) + } + fc.FindLabel(fc.Block.Parent, gotoLabel, i) + } +} + +func (fc *funcContext) ResolveForwardGoto(target *gotoLabelDesc) { + for i := fc.Block.firstGotoIndex; i <= fc.gotosCount; i++ { + gotoLabel, ok := fc.unresolvedGotos[i] + if !ok { + continue + } + if gotoLabel.Name == target.Name { + fc.ResolveGoto(gotoLabel, target, i) + } + } +} + +func (fc *funcContext) NewLabel() int { + ret := fc.labelId + fc.labelId++ + return ret +} + +func (fc *funcContext) SetLabelPc(label int, pc int) { + fc.labelPc[label] = pc +} + +func (fc *funcContext) GetLabelPc(label int) int { + return fc.labelPc[label] +} + +func (fc *funcContext) ConstIndex(value LValue) int { + ctype := value.Type() + for i, lv := range fc.Proto.Constants { + if lv.Type() == ctype && lv == value { + return i + } + } + fc.Proto.Constants = append(fc.Proto.Constants, value) + v := len(fc.Proto.Constants) - 1 + if v > opMaxArgBx { + raiseCompileError(fc, fc.Proto.LineDefined, "too many constants") + } + return v +} +func (fc *funcContext) BlockLocalVarsCount() int { + count := 0 + for block := fc.Block; block != nil; block = block.Parent { + count += len(block.LocalVars.Names()) + } + return count +} + +func (fc *funcContext) RegisterLocalVar(name string) int { + ret := fc.Block.LocalVars.Register(name) + fc.Proto.DbgLocals = append(fc.Proto.DbgLocals, &DbgLocalInfo{Name: name, StartPc: fc.Code.LastPC() + 1}) + fc.SetRegTop(fc.RegTop() + 1) + return ret +} + +func (fc *funcContext) FindLocalVarAndBlock(name string) (int, *codeBlock) { + for block := fc.Block; block != nil; block = block.Parent { + if index := block.LocalVars.Find(name); index > -1 { + return index, block + } + } + return -1, nil +} + +func (fc *funcContext) FindLocalVar(name string) int { + idx, _ := fc.FindLocalVarAndBlock(name) + return idx +} + +func (fc *funcContext) LocalVars() []varNamePoolValue { + result := make([]varNamePoolValue, 0, 32) + for _, block := range fc.Blocks { + result = append(result, block.LocalVars.List()...) + } + return result +} + +func (fc *funcContext) EnterBlock(blabel int, pos ast.PositionHolder) { + fc.Block = newCodeBlock(newVarNamePool(fc.RegTop()), blabel, fc.Block, pos, fc.gotosCount) + fc.Blocks = append(fc.Blocks, fc.Block) +} + +func (fc *funcContext) CloseUpvalues() int { + n := -1 + if fc.Block.RefUpvalue { + n = fc.Block.Parent.LocalVars.LastIndex() + fc.Code.AddABC(OP_CLOSE, n, 0, 0, fc.Block.LastLine) + } + return n +} + +func (fc *funcContext) LeaveBlock() int { + closed := fc.CloseUpvalues() + fc.EndScope() + + if fc.Block.Parent != nil { + fc.ResolveCurrentBlockGotosWithParentBlock() + } + fc.Block = fc.Block.Parent + fc.SetRegTop(fc.Block.LocalVars.LastIndex()) + return closed +} + +func (fc *funcContext) EndScope() { + for _, vr := range fc.Block.LocalVars.List() { + fc.Proto.DbgLocals[vr.Index].EndPc = fc.Code.LastPC() + } +} + +func (fc *funcContext) SetRegTop(top int) { + if top > maxRegisters { + raiseCompileError(fc, fc.Proto.LineDefined, "too many local variables") + } + fc.regTop = top +} + +func (fc *funcContext) RegTop() int { + return fc.regTop +} + +/* FuncContext }}} */ + +func compileChunk(context *funcContext, chunk []ast.Stmt, untilFollows bool) { // {{{ + for i, stmt := range chunk { + lastStmt := true + for j := i + 1; j < len(chunk); j++ { + _, ok := chunk[j].(*ast.LabelStmt) + if !ok { + lastStmt = false + break + } + } + compileStmt(context, stmt, lastStmt && !untilFollows) + } +} // }}} + +func compileBlock(context *funcContext, chunk []ast.Stmt) { // {{{ + if len(chunk) == 0 { + return + } + ph := &ast.Node{} + ph.SetLine(sline(chunk[0])) + ph.SetLastLine(eline(chunk[len(chunk)-1])) + context.EnterBlock(labelNoJump, ph) + for i, stmt := range chunk { + lastStmt := true + for j := i + 1; j < len(chunk); j++ { + _, ok := chunk[j].(*ast.LabelStmt) + if !ok { + lastStmt = false + break + } + } + compileStmt(context, stmt, lastStmt) + } + context.LeaveBlock() +} // }}} + +func compileStmt(context *funcContext, stmt ast.Stmt, isLastStmt bool) { // {{{ + switch st := stmt.(type) { + case *ast.AssignStmt: + compileAssignStmt(context, st) + case *ast.LocalAssignStmt: + compileLocalAssignStmt(context, st) + case *ast.FuncCallStmt: + compileFuncCallExpr(context, context.RegTop(), st.Expr.(*ast.FuncCallExpr), ecnone(-1)) + case *ast.DoBlockStmt: + context.EnterBlock(labelNoJump, st) + compileChunk(context, st.Stmts, false) + context.LeaveBlock() + case *ast.WhileStmt: + compileWhileStmt(context, st) + case *ast.RepeatStmt: + compileRepeatStmt(context, st) + case *ast.FuncDefStmt: + compileFuncDefStmt(context, st) + case *ast.ReturnStmt: + compileReturnStmt(context, st) + case *ast.IfStmt: + compileIfStmt(context, st) + case *ast.BreakStmt: + compileBreakStmt(context, st) + case *ast.NumberForStmt: + compileNumberForStmt(context, st) + case *ast.GenericForStmt: + compileGenericForStmt(context, st) + case *ast.LabelStmt: + compileLabelStmt(context, st, isLastStmt) + case *ast.GotoStmt: + compileGotoStmt(context, st) + } +} // }}} + +func compileAssignStmtLeft(context *funcContext, stmt *ast.AssignStmt) (int, []*assigncontext) { // {{{ + reg := context.RegTop() + acs := make([]*assigncontext, 0, len(stmt.Lhs)) + for _, lhs := range stmt.Lhs { + switch st := lhs.(type) { + case *ast.IdentExpr: + identtype := getIdentRefType(context, context, st) + ec := &expcontext{identtype, regNotDefined, 0} + switch identtype { + case ecGlobal: + context.ConstIndex(LString(st.Value)) + case ecUpvalue: + context.Upvalues.RegisterUnique(st.Value) + case ecLocal: + ec.reg = context.FindLocalVar(st.Value) + } + acs = append(acs, &assigncontext{ec, 0, 0, false, false}) + case *ast.AttrGetExpr: + ac := &assigncontext{&expcontext{ecTable, regNotDefined, 0}, 0, 0, false, false} + compileExprWithKMVPropagation(context, st.Object, ®, &ac.ec.reg) + ac.keyrk = reg + reg += compileExpr(context, reg, st.Key, ecnone(0)) + if _, ok := st.Key.(*ast.StringExpr); ok { + ac.keyks = true + } + acs = append(acs, ac) + + default: + panic("invalid left expression.") + } + } + return reg, acs +} // }}} + +func compileAssignStmtRight(context *funcContext, stmt *ast.AssignStmt, reg int, acs []*assigncontext) (int, []*assigncontext) { // {{{ + lennames := len(stmt.Lhs) + lenexprs := len(stmt.Rhs) + namesassigned := 0 + + for namesassigned < lennames { + ac := acs[namesassigned] + ec := ac.ec + var expr ast.Expr = nil + if namesassigned >= lenexprs { + expr = &ast.NilExpr{} + expr.SetLine(sline(stmt.Lhs[namesassigned])) + expr.SetLastLine(eline(stmt.Lhs[namesassigned])) + } else if isVarArgReturnExpr(stmt.Rhs[namesassigned]) && (lenexprs-namesassigned-1) <= 0 { + varargopt := lennames - namesassigned - 1 + regstart := reg + reginc := compileExpr(context, reg, stmt.Rhs[namesassigned], ecnone(varargopt)) + reg += reginc + for i := namesassigned; i < namesassigned+int(reginc); i++ { + acs[i].needmove = true + if acs[i].ec.ctype == ecTable { + acs[i].valuerk = regstart + (i - namesassigned) + } + } + namesassigned = lennames + continue + } + + if expr == nil { + expr = stmt.Rhs[namesassigned] + } + idx := reg + reginc := compileExpr(context, reg, expr, ec) + if ec.ctype == ecTable { + if _, ok := expr.(*ast.LogicalOpExpr); !ok { + context.Code.PropagateKMV(context.RegTop(), &ac.valuerk, ®, reginc) + } else { + ac.valuerk = idx + reg += reginc + } + } else { + ac.needmove = reginc != 0 + reg += reginc + } + namesassigned += 1 + } + + rightreg := reg - 1 + + // extra right exprs + for i := namesassigned; i < lenexprs; i++ { + varargopt := -1 + if i != lenexprs-1 { + varargopt = 0 + } + reg += compileExpr(context, reg, stmt.Rhs[i], ecnone(varargopt)) + } + return rightreg, acs +} // }}} + +func compileAssignStmt(context *funcContext, stmt *ast.AssignStmt) { // {{{ + code := context.Code + lennames := len(stmt.Lhs) + reg, acs := compileAssignStmtLeft(context, stmt) + reg, acs = compileAssignStmtRight(context, stmt, reg, acs) + + for i := lennames - 1; i >= 0; i-- { + ex := stmt.Lhs[i] + switch acs[i].ec.ctype { + case ecLocal: + if acs[i].needmove { + code.AddABC(OP_MOVE, context.FindLocalVar(ex.(*ast.IdentExpr).Value), reg, 0, sline(ex)) + reg -= 1 + } + case ecGlobal: + code.AddABx(OP_SETGLOBAL, reg, context.ConstIndex(LString(ex.(*ast.IdentExpr).Value)), sline(ex)) + reg -= 1 + case ecUpvalue: + code.AddABC(OP_SETUPVAL, reg, context.Upvalues.RegisterUnique(ex.(*ast.IdentExpr).Value), 0, sline(ex)) + reg -= 1 + case ecTable: + opcode := OP_SETTABLE + if acs[i].keyks { + opcode = OP_SETTABLEKS + } + code.AddABC(opcode, acs[i].ec.reg, acs[i].keyrk, acs[i].valuerk, sline(ex)) + if !opIsK(acs[i].valuerk) { + reg -= 1 + } + } + } +} // }}} + +func compileRegAssignment(context *funcContext, names []string, exprs []ast.Expr, reg int, nvars int, line int) { // {{{ + lennames := len(names) + lenexprs := len(exprs) + namesassigned := 0 + ec := &expcontext{} + + for namesassigned < lennames && namesassigned < lenexprs { + if isVarArgReturnExpr(exprs[namesassigned]) && (lenexprs-namesassigned-1) <= 0 { + + varargopt := nvars - namesassigned + ecupdate(ec, ecVararg, reg, varargopt-1) + compileExpr(context, reg, exprs[namesassigned], ec) + reg += varargopt + namesassigned = lennames + } else { + ecupdate(ec, ecLocal, reg, 0) + compileExpr(context, reg, exprs[namesassigned], ec) + reg += 1 + namesassigned += 1 + } + } + + // extra left names + if lennames > namesassigned { + restleft := lennames - namesassigned - 1 + context.Code.AddLoadNil(reg, reg+restleft, line) + reg += restleft + } + + // extra right exprs + for i := namesassigned; i < lenexprs; i++ { + varargopt := -1 + if i != lenexprs-1 { + varargopt = 0 + } + ecupdate(ec, ecNone, reg, varargopt) + reg += compileExpr(context, reg, exprs[i], ec) + } +} // }}} + +func compileLocalAssignStmt(context *funcContext, stmt *ast.LocalAssignStmt) { // {{{ + reg := context.RegTop() + if len(stmt.Names) == 1 && len(stmt.Exprs) == 1 { + if _, ok := stmt.Exprs[0].(*ast.FunctionExpr); ok { + context.RegisterLocalVar(stmt.Names[0]) + compileRegAssignment(context, stmt.Names, stmt.Exprs, reg, len(stmt.Names), sline(stmt)) + return + } + } + + compileRegAssignment(context, stmt.Names, stmt.Exprs, reg, len(stmt.Names), sline(stmt)) + for _, name := range stmt.Names { + context.RegisterLocalVar(name) + } +} // }}} + +func compileReturnStmt(context *funcContext, stmt *ast.ReturnStmt) { // {{{ + lenexprs := len(stmt.Exprs) + code := context.Code + reg := context.RegTop() + a := reg + lastisvaarg := false + + if lenexprs == 1 { + switch ex := stmt.Exprs[0].(type) { + case *ast.IdentExpr: + if idx := context.FindLocalVar(ex.Value); idx > -1 { + code.AddABC(OP_RETURN, idx, 2, 0, sline(stmt)) + return + } + case *ast.FuncCallExpr: + if ex.AdjustRet { // return (func()) + reg += compileExpr(context, reg, ex, ecnone(0)) + } else { + reg += compileExpr(context, reg, ex, ecnone(-2)) + code.SetOpCode(code.LastPC(), OP_TAILCALL) + } + code.AddABC(OP_RETURN, a, 0, 0, sline(stmt)) + return + } + } + + for i, expr := range stmt.Exprs { + if i == lenexprs-1 && isVarArgReturnExpr(expr) { + compileExpr(context, reg, expr, ecnone(-2)) + lastisvaarg = true + } else { + reg += compileExpr(context, reg, expr, ecnone(0)) + } + } + count := reg - a + 1 + if lastisvaarg { + count = 0 + } + context.Code.AddABC(OP_RETURN, a, count, 0, sline(stmt)) +} // }}} + +func compileIfStmt(context *funcContext, stmt *ast.IfStmt) { // {{{ + thenlabel := context.NewLabel() + elselabel := context.NewLabel() + endlabel := context.NewLabel() + + compileBranchCondition(context, context.RegTop(), stmt.Condition, thenlabel, elselabel, false) + context.SetLabelPc(thenlabel, context.Code.LastPC()) + compileBlock(context, stmt.Then) + if len(stmt.Else) > 0 { + context.Code.AddASbx(OP_JMP, 0, endlabel, sline(stmt)) + } + context.SetLabelPc(elselabel, context.Code.LastPC()) + if len(stmt.Else) > 0 { + compileBlock(context, stmt.Else) + context.SetLabelPc(endlabel, context.Code.LastPC()) + } + +} // }}} + +func compileBranchCondition(context *funcContext, reg int, expr ast.Expr, thenlabel, elselabel int, hasnextcond bool) { // {{{ + // TODO folding constants? + code := context.Code + flip := 0 + jumplabel := elselabel + if hasnextcond { + flip = 1 + jumplabel = thenlabel + } + + switch ex := expr.(type) { + case *ast.FalseExpr, *ast.NilExpr: + if !hasnextcond { + code.AddASbx(OP_JMP, 0, elselabel, sline(expr)) + return + } + case *ast.TrueExpr, *ast.NumberExpr, *ast.StringExpr: + if !hasnextcond { + return + } + case *ast.UnaryNotOpExpr: + compileBranchCondition(context, reg, ex.Expr, elselabel, thenlabel, !hasnextcond) + return + case *ast.LogicalOpExpr: + switch ex.Operator { + case "and": + nextcondlabel := context.NewLabel() + compileBranchCondition(context, reg, ex.Lhs, nextcondlabel, elselabel, false) + context.SetLabelPc(nextcondlabel, context.Code.LastPC()) + compileBranchCondition(context, reg, ex.Rhs, thenlabel, elselabel, hasnextcond) + case "or": + nextcondlabel := context.NewLabel() + compileBranchCondition(context, reg, ex.Lhs, thenlabel, nextcondlabel, true) + context.SetLabelPc(nextcondlabel, context.Code.LastPC()) + compileBranchCondition(context, reg, ex.Rhs, thenlabel, elselabel, hasnextcond) + } + return + case *ast.RelationalOpExpr: + compileRelationalOpExprAux(context, reg, ex, flip, jumplabel) + return + } + + a := reg + compileExprWithMVPropagation(context, expr, ®, &a) + code.AddABC(OP_TEST, a, 0, 0^flip, sline(expr)) + code.AddASbx(OP_JMP, 0, jumplabel, sline(expr)) +} // }}} + +func compileWhileStmt(context *funcContext, stmt *ast.WhileStmt) { // {{{ + thenlabel := context.NewLabel() + elselabel := context.NewLabel() + condlabel := context.NewLabel() + + context.SetLabelPc(condlabel, context.Code.LastPC()) + compileBranchCondition(context, context.RegTop(), stmt.Condition, thenlabel, elselabel, false) + context.SetLabelPc(thenlabel, context.Code.LastPC()) + context.EnterBlock(elselabel, stmt) + compileChunk(context, stmt.Stmts, false) + context.CloseUpvalues() + context.Code.AddASbx(OP_JMP, 0, condlabel, eline(stmt)) + context.LeaveBlock() + context.SetLabelPc(elselabel, context.Code.LastPC()) +} // }}} + +func compileRepeatStmt(context *funcContext, stmt *ast.RepeatStmt) { // {{{ + initlabel := context.NewLabel() + thenlabel := context.NewLabel() + elselabel := context.NewLabel() + + context.SetLabelPc(initlabel, context.Code.LastPC()) + context.SetLabelPc(elselabel, context.Code.LastPC()) + context.EnterBlock(thenlabel, stmt) + compileChunk(context, stmt.Stmts, true) + compileBranchCondition(context, context.RegTop(), stmt.Condition, thenlabel, elselabel, false) + + context.SetLabelPc(thenlabel, context.Code.LastPC()) + n := context.LeaveBlock() + + if n > -1 { + label := context.NewLabel() + context.Code.AddASbx(OP_JMP, 0, label, eline(stmt)) + context.SetLabelPc(elselabel, context.Code.LastPC()) + context.Code.AddABC(OP_CLOSE, n, 0, 0, eline(stmt)) + context.Code.AddASbx(OP_JMP, 0, initlabel, eline(stmt)) + context.SetLabelPc(label, context.Code.LastPC()) + } + +} // }}} + +func compileBreakStmt(context *funcContext, stmt *ast.BreakStmt) { // {{{ + for block := context.Block; block != nil; block = block.Parent { + if label := block.BreakLabel; label != labelNoJump { + if block.RefUpvalue { + context.Code.AddABC(OP_CLOSE, block.Parent.LocalVars.LastIndex(), 0, 0, sline(stmt)) + } + context.Code.AddASbx(OP_JMP, 0, label, sline(stmt)) + return + } + } + raiseCompileError(context, sline(stmt), "no loop to break") +} // }}} + +func compileFuncDefStmt(context *funcContext, stmt *ast.FuncDefStmt) { // {{{ + if stmt.Name.Func == nil { + reg := context.RegTop() + var treg, kreg int + compileExprWithKMVPropagation(context, stmt.Name.Receiver, ®, &treg) + kreg = loadRk(context, ®, stmt.Func, LString(stmt.Name.Method)) + compileExpr(context, reg, stmt.Func, ecfuncdef) + context.Code.AddABC(OP_SETTABLE, treg, kreg, reg, sline(stmt.Name.Receiver)) + } else { + astmt := &ast.AssignStmt{Lhs: []ast.Expr{stmt.Name.Func}, Rhs: []ast.Expr{stmt.Func}} + astmt.SetLine(sline(stmt.Func)) + astmt.SetLastLine(eline(stmt.Func)) + compileAssignStmt(context, astmt) + } +} // }}} + +func compileNumberForStmt(context *funcContext, stmt *ast.NumberForStmt) { // {{{ + code := context.Code + endlabel := context.NewLabel() + ec := &expcontext{} + + context.EnterBlock(endlabel, stmt) + reg := context.RegTop() + rindex := context.RegisterLocalVar("(for index)") + ecupdate(ec, ecLocal, rindex, 0) + compileExpr(context, reg, stmt.Init, ec) + + reg = context.RegTop() + rlimit := context.RegisterLocalVar("(for limit)") + ecupdate(ec, ecLocal, rlimit, 0) + compileExpr(context, reg, stmt.Limit, ec) + + reg = context.RegTop() + rstep := context.RegisterLocalVar("(for step)") + if stmt.Step == nil { + stmt.Step = &ast.NumberExpr{Value: "1"} + stmt.Step.SetLine(sline(stmt.Init)) + } + ecupdate(ec, ecLocal, rstep, 0) + compileExpr(context, reg, stmt.Step, ec) + + code.AddASbx(OP_FORPREP, rindex, 0, sline(stmt)) + + context.RegisterLocalVar(stmt.Name) + + bodypc := code.LastPC() + compileChunk(context, stmt.Stmts, false) + + context.LeaveBlock() + + flpc := code.LastPC() + code.AddASbx(OP_FORLOOP, rindex, bodypc-(flpc+1), sline(stmt)) + + context.SetLabelPc(endlabel, code.LastPC()) + code.SetSbx(bodypc, flpc-bodypc) + +} // }}} + +func compileGenericForStmt(context *funcContext, stmt *ast.GenericForStmt) { // {{{ + code := context.Code + endlabel := context.NewLabel() + bodylabel := context.NewLabel() + fllabel := context.NewLabel() + nnames := len(stmt.Names) + + context.EnterBlock(endlabel, stmt) + rgen := context.RegisterLocalVar("(for generator)") + context.RegisterLocalVar("(for state)") + context.RegisterLocalVar("(for control)") + + compileRegAssignment(context, stmt.Names, stmt.Exprs, context.RegTop()-3, 3, sline(stmt)) + + code.AddASbx(OP_JMP, 0, fllabel, sline(stmt)) + + for _, name := range stmt.Names { + context.RegisterLocalVar(name) + } + + context.SetLabelPc(bodylabel, code.LastPC()) + compileChunk(context, stmt.Stmts, false) + + context.LeaveBlock() + + context.SetLabelPc(fllabel, code.LastPC()) + code.AddABC(OP_TFORLOOP, rgen, 0, nnames, sline(stmt)) + code.AddASbx(OP_JMP, 0, bodylabel, sline(stmt)) + + context.SetLabelPc(endlabel, code.LastPC()) +} // }}} + +func compileLabelStmt(context *funcContext, stmt *ast.LabelStmt, isLastStmt bool) { // {{{ + labelId := context.NewLabel() + label := newLabelDesc(labelId, stmt.Name, context.Code.LastPC(), sline(stmt), context.BlockLocalVarsCount()) + context.AddNamedLabel(label) + if isLastStmt { + label.SetNumActiveLocalVars(context.Block.Parent.LocalVarsCount()) + } + context.ResolveForwardGoto(label) +} // }}} + +func compileGotoStmt(context *funcContext, stmt *ast.GotoStmt) { // {{{ + context.Code.AddABC(OP_CLOSE, 0, 0, 0, sline(stmt)) + context.Code.AddASbx(OP_JMP, 0, labelNoJump, sline(stmt)) + label := newLabelDesc(-1, stmt.Label, context.Code.LastPC(), sline(stmt), context.BlockLocalVarsCount()) + context.AddUnresolvedGoto(label) + context.FindLabel(context.Block, label, context.gotosCount-1) +} // }}} + +func compileExpr(context *funcContext, reg int, expr ast.Expr, ec *expcontext) int { // {{{ + code := context.Code + sreg := savereg(ec, reg) + sused := 1 + if sreg < reg { + sused = 0 + } + + switch ex := expr.(type) { + case *ast.StringExpr: + code.AddABx(OP_LOADK, sreg, context.ConstIndex(LString(ex.Value)), sline(ex)) + return sused + case *ast.NumberExpr: + num, err := parseNumber(ex.Value) + if err != nil { + num = LNumber(math.NaN()) + } + code.AddABx(OP_LOADK, sreg, context.ConstIndex(num), sline(ex)) + return sused + case *constLValueExpr: + code.AddABx(OP_LOADK, sreg, context.ConstIndex(ex.Value), sline(ex)) + return sused + case *ast.NilExpr: + code.AddLoadNil(sreg, sreg, sline(ex)) + return sused + case *ast.FalseExpr: + code.AddABC(OP_LOADBOOL, sreg, 0, 0, sline(ex)) + return sused + case *ast.TrueExpr: + code.AddABC(OP_LOADBOOL, sreg, 1, 0, sline(ex)) + return sused + case *ast.IdentExpr: + switch getIdentRefType(context, context, ex) { + case ecGlobal: + code.AddABx(OP_GETGLOBAL, sreg, context.ConstIndex(LString(ex.Value)), sline(ex)) + case ecUpvalue: + code.AddABC(OP_GETUPVAL, sreg, context.Upvalues.RegisterUnique(ex.Value), 0, sline(ex)) + case ecLocal: + b := context.FindLocalVar(ex.Value) + code.AddABC(OP_MOVE, sreg, b, 0, sline(ex)) + } + return sused + case *ast.Comma3Expr: + if context.Proto.IsVarArg == 0 { + raiseCompileError(context, sline(ex), "cannot use '...' outside a vararg function") + } + context.Proto.IsVarArg &= ^VarArgNeedsArg + code.AddABC(OP_VARARG, sreg, 2+ec.varargopt, 0, sline(ex)) + if context.RegTop() > (sreg+2+ec.varargopt) || ec.varargopt < -1 { + return 0 + } + return (sreg + 1 + ec.varargopt) - reg + case *ast.AttrGetExpr: + a := sreg + b := reg + compileExprWithMVPropagation(context, ex.Object, ®, &b) + c := reg + compileExprWithKMVPropagation(context, ex.Key, ®, &c) + opcode := OP_GETTABLE + if _, ok := ex.Key.(*ast.StringExpr); ok { + opcode = OP_GETTABLEKS + } + code.AddABC(opcode, a, b, c, sline(ex)) + return sused + case *ast.TableExpr: + compileTableExpr(context, reg, ex, ec) + return 1 + case *ast.ArithmeticOpExpr: + compileArithmeticOpExpr(context, reg, ex, ec) + return sused + case *ast.StringConcatOpExpr: + compileStringConcatOpExpr(context, reg, ex, ec) + return sused + case *ast.UnaryMinusOpExpr, *ast.UnaryNotOpExpr, *ast.UnaryLenOpExpr: + compileUnaryOpExpr(context, reg, ex, ec) + return sused + case *ast.RelationalOpExpr: + compileRelationalOpExpr(context, reg, ex, ec) + return sused + case *ast.LogicalOpExpr: + compileLogicalOpExpr(context, reg, ex, ec) + return sused + case *ast.FuncCallExpr: + return compileFuncCallExpr(context, reg, ex, ec) + case *ast.FunctionExpr: + childcontext := newFuncContext(context.Proto.SourceName, context) + compileFunctionExpr(childcontext, ex, ec) + protono := len(context.Proto.FunctionPrototypes) + context.Proto.FunctionPrototypes = append(context.Proto.FunctionPrototypes, childcontext.Proto) + code.AddABx(OP_CLOSURE, sreg, protono, sline(ex)) + for _, upvalue := range childcontext.Upvalues.List() { + localidx, block := context.FindLocalVarAndBlock(upvalue.Name) + if localidx > -1 { + code.AddABC(OP_MOVE, 0, localidx, 0, sline(ex)) + block.RefUpvalue = true + } else { + upvalueidx := context.Upvalues.Find(upvalue.Name) + if upvalueidx < 0 { + upvalueidx = context.Upvalues.RegisterUnique(upvalue.Name) + } + code.AddABC(OP_GETUPVAL, 0, upvalueidx, 0, sline(ex)) + } + } + return sused + default: + panic(fmt.Sprintf("expr %v not implemented.", reflect.TypeOf(ex).Elem().Name())) + } + +} // }}} + +func compileExprWithPropagation(context *funcContext, expr ast.Expr, reg *int, save *int, propergator func(int, *int, *int, int)) { // {{{ + reginc := compileExpr(context, *reg, expr, ecnone(0)) + if _, ok := expr.(*ast.LogicalOpExpr); ok { + *save = *reg + *reg = *reg + reginc + } else { + propergator(context.RegTop(), save, reg, reginc) + } +} // }}} + +func compileExprWithKMVPropagation(context *funcContext, expr ast.Expr, reg *int, save *int) { // {{{ + compileExprWithPropagation(context, expr, reg, save, context.Code.PropagateKMV) +} // }}} + +func compileExprWithMVPropagation(context *funcContext, expr ast.Expr, reg *int, save *int) { // {{{ + compileExprWithPropagation(context, expr, reg, save, context.Code.PropagateMV) +} // }}} + +func constFold(exp ast.Expr) ast.Expr { // {{{ + switch expr := exp.(type) { + case *ast.ArithmeticOpExpr: + lvalue, lisconst := lnumberValue(constFold(expr.Lhs)) + rvalue, risconst := lnumberValue(constFold(expr.Rhs)) + if lisconst && risconst { + switch expr.Operator { + case "+": + return &constLValueExpr{Value: lvalue + rvalue} + case "-": + return &constLValueExpr{Value: lvalue - rvalue} + case "*": + return &constLValueExpr{Value: lvalue * rvalue} + case "/": + return &constLValueExpr{Value: lvalue / rvalue} + case "%": + return &constLValueExpr{Value: luaModulo(lvalue, rvalue)} + case "^": + return &constLValueExpr{Value: LNumber(math.Pow(float64(lvalue), float64(rvalue)))} + default: + panic(fmt.Sprintf("unknown binop: %v", expr.Operator)) + } + } else { + return expr + } + case *ast.UnaryMinusOpExpr: + expr.Expr = constFold(expr.Expr) + if value, ok := lnumberValue(expr.Expr); ok { + return &constLValueExpr{Value: LNumber(-value)} + } + return expr + default: + + return exp + } +} // }}} + +func compileFunctionExpr(context *funcContext, funcexpr *ast.FunctionExpr, ec *expcontext) { // {{{ + context.Proto.LineDefined = sline(funcexpr) + context.Proto.LastLineDefined = eline(funcexpr) + if len(funcexpr.ParList.Names) > maxRegisters { + raiseCompileError(context, context.Proto.LineDefined, "register overflow") + } + context.Proto.NumParameters = uint8(len(funcexpr.ParList.Names)) + if ec.ctype == ecMethod { + context.Proto.NumParameters += 1 + context.RegisterLocalVar("self") + } + for _, name := range funcexpr.ParList.Names { + context.RegisterLocalVar(name) + } + if funcexpr.ParList.HasVargs { + if CompatVarArg { + context.Proto.IsVarArg = VarArgHasArg | VarArgNeedsArg + if context.Parent != nil { + context.RegisterLocalVar("arg") + } + } + context.Proto.IsVarArg |= VarArgIsVarArg + } + + compileChunk(context, funcexpr.Stmts, false) + + context.Code.AddABC(OP_RETURN, 0, 1, 0, eline(funcexpr)) + context.EndScope() + context.CheckUnresolvedGoto() + context.Proto.Code = context.Code.List() + context.Proto.DbgSourcePositions = context.Code.PosList() + context.Proto.DbgUpvalues = context.Upvalues.Names() + context.Proto.NumUpvalues = uint8(len(context.Proto.DbgUpvalues)) + for _, clv := range context.Proto.Constants { + sv := "" + if slv, ok := clv.(LString); ok { + sv = string(slv) + } + context.Proto.stringConstants = append(context.Proto.stringConstants, sv) + } + patchCode(context) +} // }}} + +func compileTableExpr(context *funcContext, reg int, ex *ast.TableExpr, ec *expcontext) { // {{{ + code := context.Code + /* + tablereg := savereg(ec, reg) + if tablereg == reg { + reg += 1 + } + */ + tablereg := reg + reg++ + code.AddABC(OP_NEWTABLE, tablereg, 0, 0, sline(ex)) + tablepc := code.LastPC() + regbase := reg + + arraycount := 0 + lastvararg := false + for i, field := range ex.Fields { + islast := i == len(ex.Fields)-1 + if field.Key == nil { + if islast && isVarArgReturnExpr(field.Value) { + reg += compileExpr(context, reg, field.Value, ecnone(-2)) + lastvararg = true + } else { + reg += compileExpr(context, reg, field.Value, ecnone(0)) + arraycount += 1 + } + } else { + regorg := reg + b := reg + compileExprWithKMVPropagation(context, field.Key, ®, &b) + c := reg + compileExprWithKMVPropagation(context, field.Value, ®, &c) + opcode := OP_SETTABLE + if _, ok := field.Key.(*ast.StringExpr); ok { + opcode = OP_SETTABLEKS + } + code.AddABC(opcode, tablereg, b, c, sline(ex)) + reg = regorg + } + flush := arraycount % FieldsPerFlush + if (arraycount != 0 && (flush == 0 || islast)) || lastvararg { + reg = regbase + num := flush + if num == 0 { + num = FieldsPerFlush + } + c := (arraycount-1)/FieldsPerFlush + 1 + b := num + if islast && isVarArgReturnExpr(field.Value) { + b = 0 + } + line := field.Value + if field.Key != nil { + line = field.Key + } + if c > 511 { + c = 0 + } + code.AddABC(OP_SETLIST, tablereg, b, c, sline(line)) + if c == 0 { + code.Add(uint32(c), sline(line)) + } + } + } + code.SetB(tablepc, int2Fb(arraycount)) + code.SetC(tablepc, int2Fb(len(ex.Fields)-arraycount)) + if shouldmove(ec, tablereg) { + code.AddABC(OP_MOVE, ec.reg, tablereg, 0, sline(ex)) + } +} // }}} + +func compileArithmeticOpExpr(context *funcContext, reg int, expr *ast.ArithmeticOpExpr, ec *expcontext) { // {{{ + exp := constFold(expr) + if ex, ok := exp.(*constLValueExpr); ok { + exp.SetLine(sline(expr)) + compileExpr(context, reg, ex, ec) + return + } + expr, _ = exp.(*ast.ArithmeticOpExpr) + a := savereg(ec, reg) + b := reg + compileExprWithKMVPropagation(context, expr.Lhs, ®, &b) + c := reg + compileExprWithKMVPropagation(context, expr.Rhs, ®, &c) + + op := 0 + switch expr.Operator { + case "+": + op = OP_ADD + case "-": + op = OP_SUB + case "*": + op = OP_MUL + case "/": + op = OP_DIV + case "%": + op = OP_MOD + case "^": + op = OP_POW + } + context.Code.AddABC(op, a, b, c, sline(expr)) +} // }}} + +func compileStringConcatOpExpr(context *funcContext, reg int, expr *ast.StringConcatOpExpr, ec *expcontext) { // {{{ + code := context.Code + crange := 1 + for current := expr.Rhs; current != nil; { + if ex, ok := current.(*ast.StringConcatOpExpr); ok { + crange += 1 + current = ex.Rhs + } else { + current = nil + } + } + a := savereg(ec, reg) + basereg := reg + reg += compileExpr(context, reg, expr.Lhs, ecnone(0)) + reg += compileExpr(context, reg, expr.Rhs, ecnone(0)) + for pc := code.LastPC(); pc != 0 && opGetOpCode(code.At(pc)) == OP_CONCAT; pc-- { + code.Pop() + } + code.AddABC(OP_CONCAT, a, basereg, basereg+crange, sline(expr)) +} // }}} + +func compileUnaryOpExpr(context *funcContext, reg int, expr ast.Expr, ec *expcontext) { // {{{ + opcode := 0 + code := context.Code + var operandexpr ast.Expr + switch ex := expr.(type) { + case *ast.UnaryMinusOpExpr: + exp := constFold(ex) + if lvexpr, ok := exp.(*constLValueExpr); ok { + exp.SetLine(sline(expr)) + compileExpr(context, reg, lvexpr, ec) + return + } + ex, _ = exp.(*ast.UnaryMinusOpExpr) + operandexpr = ex.Expr + opcode = OP_UNM + case *ast.UnaryNotOpExpr: + switch ex.Expr.(type) { + case *ast.TrueExpr: + code.AddABC(OP_LOADBOOL, savereg(ec, reg), 0, 0, sline(expr)) + return + case *ast.FalseExpr, *ast.NilExpr: + code.AddABC(OP_LOADBOOL, savereg(ec, reg), 1, 0, sline(expr)) + return + default: + opcode = OP_NOT + operandexpr = ex.Expr + } + case *ast.UnaryLenOpExpr: + opcode = OP_LEN + operandexpr = ex.Expr + } + + a := savereg(ec, reg) + b := reg + compileExprWithMVPropagation(context, operandexpr, ®, &b) + code.AddABC(opcode, a, b, 0, sline(expr)) +} // }}} + +func compileRelationalOpExprAux(context *funcContext, reg int, expr *ast.RelationalOpExpr, flip int, label int) { // {{{ + code := context.Code + b := reg + compileExprWithKMVPropagation(context, expr.Lhs, ®, &b) + c := reg + compileExprWithKMVPropagation(context, expr.Rhs, ®, &c) + switch expr.Operator { + case "<": + code.AddABC(OP_LT, 0^flip, b, c, sline(expr)) + case ">": + code.AddABC(OP_LT, 0^flip, c, b, sline(expr)) + case "<=": + code.AddABC(OP_LE, 0^flip, b, c, sline(expr)) + case ">=": + code.AddABC(OP_LE, 0^flip, c, b, sline(expr)) + case "==": + code.AddABC(OP_EQ, 0^flip, b, c, sline(expr)) + case "~=": + code.AddABC(OP_EQ, 1^flip, b, c, sline(expr)) + } + code.AddASbx(OP_JMP, 0, label, sline(expr)) +} // }}} + +func compileRelationalOpExpr(context *funcContext, reg int, expr *ast.RelationalOpExpr, ec *expcontext) { // {{{ + a := savereg(ec, reg) + code := context.Code + jumplabel := context.NewLabel() + compileRelationalOpExprAux(context, reg, expr, 1, jumplabel) + code.AddABC(OP_LOADBOOL, a, 0, 1, sline(expr)) + context.SetLabelPc(jumplabel, code.LastPC()) + code.AddABC(OP_LOADBOOL, a, 1, 0, sline(expr)) +} // }}} + +func compileLogicalOpExpr(context *funcContext, reg int, expr *ast.LogicalOpExpr, ec *expcontext) { // {{{ + a := savereg(ec, reg) + code := context.Code + endlabel := context.NewLabel() + lb := &lblabels{context.NewLabel(), context.NewLabel(), endlabel, false} + nextcondlabel := context.NewLabel() + if expr.Operator == "and" { + compileLogicalOpExprAux(context, reg, expr.Lhs, ec, nextcondlabel, endlabel, false, lb) + context.SetLabelPc(nextcondlabel, code.LastPC()) + compileLogicalOpExprAux(context, reg, expr.Rhs, ec, endlabel, endlabel, false, lb) + } else { + compileLogicalOpExprAux(context, reg, expr.Lhs, ec, endlabel, nextcondlabel, true, lb) + context.SetLabelPc(nextcondlabel, code.LastPC()) + compileLogicalOpExprAux(context, reg, expr.Rhs, ec, endlabel, endlabel, false, lb) + } + + if lb.b { + context.SetLabelPc(lb.f, code.LastPC()) + code.AddABC(OP_LOADBOOL, a, 0, 1, sline(expr)) + context.SetLabelPc(lb.t, code.LastPC()) + code.AddABC(OP_LOADBOOL, a, 1, 0, sline(expr)) + } + + lastinst := code.Last() + if opGetOpCode(lastinst) == OP_JMP && opGetArgSbx(lastinst) == endlabel { + code.Pop() + } + + context.SetLabelPc(endlabel, code.LastPC()) +} // }}} + +func compileLogicalOpExprAux(context *funcContext, reg int, expr ast.Expr, ec *expcontext, thenlabel, elselabel int, hasnextcond bool, lb *lblabels) { // {{{ + // TODO folding constants? + code := context.Code + flip := 0 + jumplabel := elselabel + if hasnextcond { + flip = 1 + jumplabel = thenlabel + } + + switch ex := expr.(type) { + case *ast.FalseExpr: + if elselabel == lb.e { + code.AddASbx(OP_JMP, 0, lb.f, sline(expr)) + lb.b = true + } else { + code.AddASbx(OP_JMP, 0, elselabel, sline(expr)) + } + return + case *ast.NilExpr: + if elselabel == lb.e { + compileExpr(context, reg, expr, ec) + code.AddASbx(OP_JMP, 0, lb.e, sline(expr)) + } else { + code.AddASbx(OP_JMP, 0, elselabel, sline(expr)) + } + return + case *ast.TrueExpr: + if thenlabel == lb.e { + code.AddASbx(OP_JMP, 0, lb.t, sline(expr)) + lb.b = true + } else { + code.AddASbx(OP_JMP, 0, thenlabel, sline(expr)) + } + return + case *ast.NumberExpr, *ast.StringExpr: + if thenlabel == lb.e { + compileExpr(context, reg, expr, ec) + code.AddASbx(OP_JMP, 0, lb.e, sline(expr)) + } else { + code.AddASbx(OP_JMP, 0, thenlabel, sline(expr)) + } + return + case *ast.LogicalOpExpr: + switch ex.Operator { + case "and": + nextcondlabel := context.NewLabel() + compileLogicalOpExprAux(context, reg, ex.Lhs, ec, nextcondlabel, elselabel, false, lb) + context.SetLabelPc(nextcondlabel, context.Code.LastPC()) + compileLogicalOpExprAux(context, reg, ex.Rhs, ec, thenlabel, elselabel, hasnextcond, lb) + case "or": + nextcondlabel := context.NewLabel() + compileLogicalOpExprAux(context, reg, ex.Lhs, ec, thenlabel, nextcondlabel, true, lb) + context.SetLabelPc(nextcondlabel, context.Code.LastPC()) + compileLogicalOpExprAux(context, reg, ex.Rhs, ec, thenlabel, elselabel, hasnextcond, lb) + } + return + case *ast.RelationalOpExpr: + if thenlabel == elselabel { + flip ^= 1 + jumplabel = lb.t + lb.b = true + } else if thenlabel == lb.e { + jumplabel = lb.t + lb.b = true + } else if elselabel == lb.e { + jumplabel = lb.f + lb.b = true + } + compileRelationalOpExprAux(context, reg, ex, flip, jumplabel) + return + } + + a := reg + sreg := savereg(ec, a) + isLastAnd := elselabel == lb.e && thenlabel != elselabel + isLastOr := thenlabel == lb.e && hasnextcond + + if ident, ok := expr.(*ast.IdentExpr); ok && (isLastAnd || isLastOr) && getIdentRefType(context, context, ident) == ecLocal { + b := context.FindLocalVar(ident.Value) + op := OP_TESTSET + if sreg == b { + op = OP_TEST + } + code.AddABC(op, sreg, b, 0^flip, sline(expr)) + } else if !hasnextcond && thenlabel == elselabel { + reg += compileExpr(context, reg, expr, &expcontext{ec.ctype, intMax(a, sreg), ec.varargopt}) + last := context.Code.Last() + if opGetOpCode(last) == OP_MOVE && opGetArgA(last) == a { + context.Code.SetA(context.Code.LastPC(), sreg) + } else { + context.Code.AddABC(OP_MOVE, sreg, a, 0, sline(expr)) + } + } else { + reg += compileExpr(context, reg, expr, ecnone(0)) + if !hasnextcond { + code.AddABC(OP_TEST, a, 0, 0^flip, sline(expr)) + } else { + code.AddABC(OP_TESTSET, sreg, a, 0^flip, sline(expr)) + } + } + code.AddASbx(OP_JMP, 0, jumplabel, sline(expr)) +} // }}} + +func compileFuncCallExpr(context *funcContext, reg int, expr *ast.FuncCallExpr, ec *expcontext) int { // {{{ + funcreg := reg + if ec.ctype == ecLocal && ec.reg == (int(context.Proto.NumParameters)-1) { + funcreg = ec.reg + reg = ec.reg + } + argc := len(expr.Args) + islastvararg := false + name := "(anonymous)" + + if expr.Func != nil { // hoge.func() + reg += compileExpr(context, reg, expr.Func, ecnone(0)) + name = getExprName(context, expr.Func) + } else { // hoge:method() + b := reg + compileExprWithMVPropagation(context, expr.Receiver, ®, &b) + c := loadRk(context, ®, expr, LString(expr.Method)) + context.Code.AddABC(OP_SELF, funcreg, b, c, sline(expr)) + // increments a register for an implicit "self" + reg = b + 1 + reg2 := funcreg + 2 + if reg2 > reg { + reg = reg2 + } + argc += 1 + name = string(expr.Method) + } + + for i, ar := range expr.Args { + islastvararg = (i == len(expr.Args)-1) && isVarArgReturnExpr(ar) + if islastvararg { + compileExpr(context, reg, ar, ecnone(-2)) + } else { + reg += compileExpr(context, reg, ar, ecnone(0)) + } + } + b := argc + 1 + if islastvararg { + b = 0 + } + context.Code.AddABC(OP_CALL, funcreg, b, ec.varargopt+2, sline(expr)) + context.Proto.DbgCalls = append(context.Proto.DbgCalls, DbgCall{Pc: context.Code.LastPC(), Name: name}) + + if ec.varargopt == 0 && shouldmove(ec, funcreg) { + context.Code.AddABC(OP_MOVE, ec.reg, funcreg, 0, sline(expr)) + return 1 + } + if context.RegTop() > (funcreg+2+ec.varargopt) || ec.varargopt < -1 { + return 0 + } + return ec.varargopt + 1 +} // }}} + +func loadRk(context *funcContext, reg *int, expr ast.Expr, cnst LValue) int { // {{{ + cindex := context.ConstIndex(cnst) + if cindex <= opMaxIndexRk { + return opRkAsk(cindex) + } else { + ret := *reg + *reg++ + context.Code.AddABx(OP_LOADK, ret, cindex, sline(expr)) + return ret + } +} // }}} + +func getIdentRefType(context *funcContext, current *funcContext, expr *ast.IdentExpr) expContextType { // {{{ + if current == nil { + return ecGlobal + } else if current.FindLocalVar(expr.Value) > -1 { + if current == context { + return ecLocal + } + return ecUpvalue + } + return getIdentRefType(context, current.Parent, expr) +} // }}} + +func getExprName(context *funcContext, expr ast.Expr) string { // {{{ + switch ex := expr.(type) { + case *ast.IdentExpr: + return ex.Value + case *ast.AttrGetExpr: + switch kex := ex.Key.(type) { + case *ast.StringExpr: + return kex.Value + } + return "?" + } + return "?" +} // }}} + +func patchCode(context *funcContext) { // {{{ + maxreg := 1 + if np := int(context.Proto.NumParameters); np > 1 { + maxreg = np + } + moven := 0 + code := context.Code.List() + for pc := 0; pc < len(code); pc++ { + inst := code[pc] + curop := opGetOpCode(inst) + switch curop { + case OP_CLOSURE: + pc += int(context.Proto.FunctionPrototypes[opGetArgBx(inst)].NumUpvalues) + moven = 0 + continue + case OP_SETGLOBAL, OP_SETUPVAL, OP_EQ, OP_LT, OP_LE, OP_TEST, + OP_TAILCALL, OP_RETURN, OP_FORPREP, OP_FORLOOP, OP_TFORLOOP, + OP_SETLIST, OP_CLOSE: + /* nothing to do */ + case OP_CALL: + if reg := opGetArgA(inst) + opGetArgC(inst) - 2; reg > maxreg { + maxreg = reg + } + case OP_VARARG: + if reg := opGetArgA(inst) + opGetArgB(inst) - 1; reg > maxreg { + maxreg = reg + } + case OP_SELF: + if reg := opGetArgA(inst) + 1; reg > maxreg { + maxreg = reg + } + case OP_LOADNIL: + if reg := opGetArgB(inst); reg > maxreg { + maxreg = reg + } + case OP_JMP: // jump to jump optimization + distance := 0 + count := 0 // avoiding infinite loops + for jmp := inst; opGetOpCode(jmp) == OP_JMP && count < 5; jmp = context.Code.At(pc + distance + 1) { + d := context.GetLabelPc(opGetArgSbx(jmp)) - pc + if d > opMaxArgSbx { + if distance == 0 { + raiseCompileError(context, context.Proto.LineDefined, "too long to jump.") + } + break + } + distance = d + count++ + } + if distance == 0 { + context.Code.SetOpCode(pc, OP_NOP) + } else { + context.Code.SetSbx(pc, distance) + } + default: + if reg := opGetArgA(inst); reg > maxreg { + maxreg = reg + } + } + + // bulk move optimization(reducing op dipatch costs) + if curop == OP_MOVE { + moven++ + } else { + if moven > 1 { + context.Code.SetOpCode(pc-moven, OP_MOVEN) + context.Code.SetC(pc-moven, intMin(moven-1, opMaxArgsC)) + } + moven = 0 + } + } + maxreg++ + if maxreg > maxRegisters { + raiseCompileError(context, context.Proto.LineDefined, "register overflow(too many local variables)") + } + context.Proto.NumUsedRegisters = uint8(maxreg) +} // }}} + +func Compile(chunk []ast.Stmt, name string) (proto *FunctionProto, err error) { // {{{ + defer func() { + if rcv := recover(); rcv != nil { + if _, ok := rcv.(*CompileError); ok { + err = rcv.(error) + } else { + panic(rcv) + } + } + }() + err = nil + parlist := &ast.ParList{HasVargs: true, Names: []string{}} + funcexpr := &ast.FunctionExpr{ParList: parlist, Stmts: chunk} + if len(chunk) > 0 { + funcexpr.SetLastLine(sline(chunk[0])) + funcexpr.SetLastLine(eline(chunk[len(chunk)-1]) + 1) + } + context := newFuncContext(name, nil) + compileFunctionExpr(context, funcexpr, ecnone(0)) + proto = context.Proto + return +} // }}} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/config.go b/scraper-go/vendor/github.com/yuin/gopher-lua/config.go new file mode 100644 index 00000000..d6321889 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/config.go @@ -0,0 +1,43 @@ +package lua + +import ( + "os" +) + +var CompatVarArg = true +var FieldsPerFlush = 50 +var RegistrySize = 256 * 20 +var RegistryGrowStep = 32 +var CallStackSize = 256 +var MaxTableGetLoop = 100 +var MaxArrayIndex = 67108864 + +type LNumber float64 + +const LNumberBit = 64 +const LNumberScanFormat = "%f" +const LuaVersion = "Lua 5.1" + +var LuaPath = "LUA_PATH" +var LuaLDir string +var LuaPathDefault string +var LuaOS string +var LuaDirSep string +var LuaPathSep = ";" +var LuaPathMark = "?" +var LuaExecDir = "!" +var LuaIgMark = "-" + +func init() { + if os.PathSeparator == '/' { // unix-like + LuaOS = "unix" + LuaLDir = "/usr/local/share/lua/5.1" + LuaDirSep = "/" + LuaPathDefault = "./?.lua;" + LuaLDir + "/?.lua;" + LuaLDir + "/?/init.lua" + } else { // windows + LuaOS = "windows" + LuaLDir = "!\\lua" + LuaDirSep = "\\" + LuaPathDefault = ".\\?.lua;" + LuaLDir + "\\?.lua;" + LuaLDir + "\\?\\init.lua" + } +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/coroutinelib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/coroutinelib.go new file mode 100644 index 00000000..d42c41a1 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/coroutinelib.go @@ -0,0 +1,112 @@ +package lua + +func OpenCoroutine(L *LState) int { + // TODO: Tie module name to contents of linit.go? + mod := L.RegisterModule(CoroutineLibName, coFuncs) + L.Push(mod) + return 1 +} + +var coFuncs = map[string]LGFunction{ + "create": coCreate, + "yield": coYield, + "resume": coResume, + "running": coRunning, + "status": coStatus, + "wrap": coWrap, +} + +func coCreate(L *LState) int { + fn := L.CheckFunction(1) + newthread, _ := L.NewThread() + base := 0 + newthread.stack.Push(callFrame{ + Fn: fn, + Pc: 0, + Base: base, + LocalBase: base + 1, + ReturnBase: base, + NArgs: 0, + NRet: MultRet, + Parent: nil, + TailCall: 0, + }) + L.Push(newthread) + return 1 +} + +func coYield(L *LState) int { + return -1 +} + +func coResume(L *LState) int { + th := L.CheckThread(1) + if L.G.CurrentThread == th { + msg := "can not resume a running thread" + if th.wrapped { + L.RaiseError(msg) + return 0 + } + L.Push(LFalse) + L.Push(LString(msg)) + return 2 + } + if th.Dead { + msg := "can not resume a dead thread" + if th.wrapped { + L.RaiseError(msg) + return 0 + } + L.Push(LFalse) + L.Push(LString(msg)) + return 2 + } + th.Parent = L + L.G.CurrentThread = th + if !th.isStarted() { + cf := th.stack.Last() + th.currentFrame = cf + th.SetTop(0) + nargs := L.GetTop() - 1 + L.XMoveTo(th, nargs) + cf.NArgs = nargs + th.initCallFrame(cf) + th.Panic = panicWithoutTraceback + } else { + nargs := L.GetTop() - 1 + L.XMoveTo(th, nargs) + } + top := L.GetTop() + threadRun(th) + return L.GetTop() - top +} + +func coRunning(L *LState) int { + if L.G.MainThread == L { + L.Push(LNil) + return 1 + } + L.Push(L.G.CurrentThread) + return 1 +} + +func coStatus(L *LState) int { + L.Push(LString(L.Status(L.CheckThread(1)))) + return 1 +} + +func wrapaux(L *LState) int { + L.Insert(L.ToThread(UpvalueIndex(1)), 1) + return coResume(L) +} + +func coWrap(L *LState) int { + coCreate(L) + L.CheckThread(L.GetTop()).wrapped = true + v := L.Get(L.GetTop()) + L.Pop(1) + L.Push(L.NewClosure(wrapaux, v)) + return 1 +} + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/debuglib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/debuglib.go new file mode 100644 index 00000000..da8f5254 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/debuglib.go @@ -0,0 +1,173 @@ +package lua + +import ( + "fmt" + "strings" +) + +func OpenDebug(L *LState) int { + dbgmod := L.RegisterModule(DebugLibName, debugFuncs) + L.Push(dbgmod) + return 1 +} + +var debugFuncs = map[string]LGFunction{ + "getfenv": debugGetFEnv, + "getinfo": debugGetInfo, + "getlocal": debugGetLocal, + "getmetatable": debugGetMetatable, + "getupvalue": debugGetUpvalue, + "setfenv": debugSetFEnv, + "setlocal": debugSetLocal, + "setmetatable": debugSetMetatable, + "setupvalue": debugSetUpvalue, + "traceback": debugTraceback, +} + +func debugGetFEnv(L *LState) int { + L.Push(L.GetFEnv(L.CheckAny(1))) + return 1 +} + +func debugGetInfo(L *LState) int { + L.CheckTypes(1, LTFunction, LTNumber) + arg1 := L.Get(1) + what := L.OptString(2, "Slunf") + var dbg *Debug + var fn LValue + var err error + var ok bool + switch lv := arg1.(type) { + case *LFunction: + dbg = &Debug{} + fn, err = L.GetInfo(">"+what, dbg, lv) + case LNumber: + dbg, ok = L.GetStack(int(lv)) + if !ok { + L.Push(LNil) + return 1 + } + fn, err = L.GetInfo(what, dbg, LNil) + } + + if err != nil { + L.Push(LNil) + return 1 + } + tbl := L.NewTable() + if len(dbg.Name) > 0 { + tbl.RawSetString("name", LString(dbg.Name)) + } else { + tbl.RawSetString("name", LNil) + } + tbl.RawSetString("what", LString(dbg.What)) + tbl.RawSetString("source", LString(dbg.Source)) + tbl.RawSetString("currentline", LNumber(dbg.CurrentLine)) + tbl.RawSetString("nups", LNumber(dbg.NUpvalues)) + tbl.RawSetString("linedefined", LNumber(dbg.LineDefined)) + tbl.RawSetString("lastlinedefined", LNumber(dbg.LastLineDefined)) + tbl.RawSetString("func", fn) + L.Push(tbl) + return 1 +} + +func debugGetLocal(L *LState) int { + level := L.CheckInt(1) + idx := L.CheckInt(2) + dbg, ok := L.GetStack(level) + if !ok { + L.ArgError(1, "level out of range") + } + name, value := L.GetLocal(dbg, idx) + if len(name) > 0 { + L.Push(LString(name)) + L.Push(value) + return 2 + } + L.Push(LNil) + return 1 +} + +func debugGetMetatable(L *LState) int { + L.Push(L.GetMetatable(L.CheckAny(1))) + return 1 +} + +func debugGetUpvalue(L *LState) int { + fn := L.CheckFunction(1) + idx := L.CheckInt(2) + name, value := L.GetUpvalue(fn, idx) + if len(name) > 0 { + L.Push(LString(name)) + L.Push(value) + return 2 + } + L.Push(LNil) + return 1 +} + +func debugSetFEnv(L *LState) int { + L.SetFEnv(L.CheckAny(1), L.CheckAny(2)) + return 0 +} + +func debugSetLocal(L *LState) int { + level := L.CheckInt(1) + idx := L.CheckInt(2) + value := L.CheckAny(3) + dbg, ok := L.GetStack(level) + if !ok { + L.ArgError(1, "level out of range") + } + name := L.SetLocal(dbg, idx, value) + if len(name) > 0 { + L.Push(LString(name)) + } else { + L.Push(LNil) + } + return 1 +} + +func debugSetMetatable(L *LState) int { + L.CheckTypes(2, LTNil, LTTable) + obj := L.Get(1) + mt := L.Get(2) + L.SetMetatable(obj, mt) + L.SetTop(1) + return 1 +} + +func debugSetUpvalue(L *LState) int { + fn := L.CheckFunction(1) + idx := L.CheckInt(2) + value := L.CheckAny(3) + name := L.SetUpvalue(fn, idx, value) + if len(name) > 0 { + L.Push(LString(name)) + } else { + L.Push(LNil) + } + return 1 +} + +func debugTraceback(L *LState) int { + msg := "" + level := L.OptInt(2, 1) + ls := L + if L.GetTop() > 0 { + if s, ok := L.Get(1).(LString); ok { + msg = string(s) + } + if l, ok := L.Get(1).(*LState); ok { + ls = l + msg = "" + } + } + + traceback := strings.TrimSpace(ls.stackTrace(level)) + if len(msg) > 0 { + traceback = fmt.Sprintf("%s\n%s", msg, traceback) + } + L.Push(LString(traceback)) + return 1 +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/function.go b/scraper-go/vendor/github.com/yuin/gopher-lua/function.go new file mode 100644 index 00000000..169e5407 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/function.go @@ -0,0 +1,193 @@ +package lua + +import ( + "fmt" + "strings" +) + +const ( + VarArgHasArg uint8 = 1 + VarArgIsVarArg uint8 = 2 + VarArgNeedsArg uint8 = 4 +) + +type DbgLocalInfo struct { + Name string + StartPc int + EndPc int +} + +type DbgCall struct { + Name string + Pc int +} + +type FunctionProto struct { + SourceName string + LineDefined int + LastLineDefined int + NumUpvalues uint8 + NumParameters uint8 + IsVarArg uint8 + NumUsedRegisters uint8 + Code []uint32 + Constants []LValue + FunctionPrototypes []*FunctionProto + + DbgSourcePositions []int + DbgLocals []*DbgLocalInfo + DbgCalls []DbgCall + DbgUpvalues []string + + stringConstants []string +} + +/* Upvalue {{{ */ + +type Upvalue struct { + next *Upvalue + reg *registry + index int + value LValue + closed bool +} + +func (uv *Upvalue) Value() LValue { + //if uv.IsClosed() { + if uv.closed || uv.reg == nil { + return uv.value + } + //return uv.reg.Get(uv.index) + return uv.reg.array[uv.index] +} + +func (uv *Upvalue) SetValue(value LValue) { + if uv.IsClosed() { + uv.value = value + } else { + uv.reg.Set(uv.index, value) + } +} + +func (uv *Upvalue) Close() { + value := uv.Value() + uv.closed = true + uv.value = value +} + +func (uv *Upvalue) IsClosed() bool { + return uv.closed || uv.reg == nil +} + +func UpvalueIndex(i int) int { + return GlobalsIndex - i +} + +/* }}} */ + +/* FunctionProto {{{ */ + +func newFunctionProto(name string) *FunctionProto { + return &FunctionProto{ + SourceName: name, + LineDefined: 0, + LastLineDefined: 0, + NumUpvalues: 0, + NumParameters: 0, + IsVarArg: 0, + NumUsedRegisters: 2, + Code: make([]uint32, 0, 128), + Constants: make([]LValue, 0, 32), + FunctionPrototypes: make([]*FunctionProto, 0, 16), + + DbgSourcePositions: make([]int, 0, 128), + DbgLocals: make([]*DbgLocalInfo, 0, 16), + DbgCalls: make([]DbgCall, 0, 128), + DbgUpvalues: make([]string, 0, 16), + + stringConstants: make([]string, 0, 32), + } +} + +func (fp *FunctionProto) String() string { + return fp.str(1, 0) +} + +func (fp *FunctionProto) str(level int, count int) string { + indent := strings.Repeat(" ", level-1) + buf := []string{} + buf = append(buf, fmt.Sprintf("%v; function [%v] definition (level %v)\n", + indent, count, level)) + buf = append(buf, fmt.Sprintf("%v; %v upvalues, %v params, %v stacks\n", + indent, fp.NumUpvalues, fp.NumParameters, fp.NumUsedRegisters)) + for reg, linfo := range fp.DbgLocals { + buf = append(buf, fmt.Sprintf("%v.local %v ; %v\n", indent, linfo.Name, reg)) + } + for reg, upvalue := range fp.DbgUpvalues { + buf = append(buf, fmt.Sprintf("%v.upvalue %v ; %v\n", indent, upvalue, reg)) + } + for reg, conzt := range fp.Constants { + buf = append(buf, fmt.Sprintf("%v.const %v ; %v\n", indent, conzt.String(), reg)) + } + buf = append(buf, "\n") + + protono := 0 + for no, code := range fp.Code { + inst := opGetOpCode(code) + if inst == OP_CLOSURE { + buf = append(buf, "\n") + buf = append(buf, fp.FunctionPrototypes[protono].str(level+1, protono)) + buf = append(buf, "\n") + protono++ + } + buf = append(buf, fmt.Sprintf("%v[%03d] %v (line:%v)\n", + indent, no+1, opToString(code), fp.DbgSourcePositions[no])) + + } + buf = append(buf, fmt.Sprintf("%v; end of function\n", indent)) + return strings.Join(buf, "") +} + +/* }}} */ + +/* LFunction {{{ */ + +func newLFunctionL(proto *FunctionProto, env *LTable, nupvalue int) *LFunction { + return &LFunction{ + IsG: false, + Env: env, + + Proto: proto, + GFunction: nil, + Upvalues: make([]*Upvalue, nupvalue), + } +} + +func newLFunctionG(gfunc LGFunction, env *LTable, nupvalue int) *LFunction { + return &LFunction{ + IsG: true, + Env: env, + + Proto: nil, + GFunction: gfunc, + Upvalues: make([]*Upvalue, nupvalue), + } +} + +func (fn *LFunction) LocalName(regno, pc int) (string, bool) { + if fn.IsG { + return "", false + } + p := fn.Proto + for i := 0; i < len(p.DbgLocals) && p.DbgLocals[i].StartPc < pc; i++ { + if pc < p.DbgLocals[i].EndPc { + regno-- + if regno == 0 { + return p.DbgLocals[i].Name, true + } + } + } + return "", false +} + +/* }}} */ diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/iolib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/iolib.go new file mode 100644 index 00000000..3f5f295c --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/iolib.go @@ -0,0 +1,749 @@ +package lua + +import ( + "bufio" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "syscall" +) + +var ioFuncs = map[string]LGFunction{ + "close": ioClose, + "flush": ioFlush, + "lines": ioLines, + "input": ioInput, + "output": ioOutput, + "open": ioOpenFile, + "popen": ioPopen, + "read": ioRead, + "type": ioType, + "tmpfile": ioTmpFile, + "write": ioWrite, +} + +const lFileClass = "FILE*" + +type lFile struct { + fp *os.File + pp *exec.Cmd + writer io.Writer + reader *bufio.Reader + stdout io.ReadCloser + closed bool +} + +type lFileType int + +const ( + lFileFile lFileType = iota + lFileProcess +) + +const fileDefOutIndex = 1 +const fileDefInIndex = 2 +const fileDefaultWriteBuffer = 4096 +const fileDefaultReadBuffer = 4096 + +func checkFile(L *LState) *lFile { + ud := L.CheckUserData(1) + if file, ok := ud.Value.(*lFile); ok { + return file + } + L.ArgError(1, "file expected") + return nil +} + +func errorIfFileIsClosed(L *LState, file *lFile) { + if file.closed { + L.ArgError(1, "file is closed") + } +} + +func newFile(L *LState, file *os.File, path string, flag int, perm os.FileMode, writable, readable bool) (*LUserData, error) { + ud := L.NewUserData() + var err error + if file == nil { + file, err = os.OpenFile(path, flag, perm) + if err != nil { + return nil, err + } + } + lfile := &lFile{fp: file, pp: nil, writer: nil, reader: nil, stdout: nil, closed: false} + ud.Value = lfile + if writable { + lfile.writer = file + } + if readable { + lfile.reader = bufio.NewReaderSize(file, fileDefaultReadBuffer) + } + L.SetMetatable(ud, L.GetTypeMetatable(lFileClass)) + return ud, nil +} + +func newProcess(L *LState, cmd string, writable, readable bool) (*LUserData, error) { + ud := L.NewUserData() + c, args := popenArgs(cmd) + pp := exec.Command(c, args...) + lfile := &lFile{fp: nil, pp: pp, writer: nil, reader: nil, stdout: nil, closed: false} + ud.Value = lfile + + var err error + if writable { + lfile.writer, err = pp.StdinPipe() + } + if readable { + lfile.stdout, err = pp.StdoutPipe() + lfile.reader = bufio.NewReaderSize(lfile.stdout, fileDefaultReadBuffer) + } + if err != nil { + return nil, err + } + err = pp.Start() + if err != nil { + return nil, err + } + + L.SetMetatable(ud, L.GetTypeMetatable(lFileClass)) + return ud, nil +} + +func (file *lFile) Type() lFileType { + if file.fp == nil { + return lFileProcess + } + return lFileFile +} + +func (file *lFile) Name() string { + switch file.Type() { + case lFileFile: + return fmt.Sprintf("file %s", file.fp.Name()) + case lFileProcess: + return fmt.Sprintf("process %s", file.pp.Path) + } + return "" +} + +func (file *lFile) AbandonReadBuffer() error { + if file.Type() == lFileFile && file.reader != nil { + _, err := file.fp.Seek(-int64(file.reader.Buffered()), 1) + if err != nil { + return err + } + file.reader = bufio.NewReaderSize(file.fp, fileDefaultReadBuffer) + } + return nil +} + +func fileDefOut(L *LState) *LUserData { + return L.Get(UpvalueIndex(1)).(*LTable).RawGetInt(fileDefOutIndex).(*LUserData) +} + +func fileDefIn(L *LState) *LUserData { + return L.Get(UpvalueIndex(1)).(*LTable).RawGetInt(fileDefInIndex).(*LUserData) +} + +func fileIsWritable(L *LState, file *lFile) int { + if file.writer == nil { + L.Push(LNil) + L.Push(LString(fmt.Sprintf("%s is opened for only reading.", file.Name()))) + L.Push(LNumber(1)) // C-Lua compatibility: Original Lua pushes errno to the stack + return 3 + } + return 0 +} + +func fileIsReadable(L *LState, file *lFile) int { + if file.reader == nil { + L.Push(LNil) + L.Push(LString(fmt.Sprintf("%s is opened for only writing.", file.Name()))) + L.Push(LNumber(1)) // C-Lua compatibility: Original Lua pushes errno to the stack + return 3 + } + return 0 +} + +var stdFiles = []struct { + name string + file *os.File + writable bool + readable bool +}{ + {"stdout", os.Stdout, true, false}, + {"stdin", os.Stdin, false, true}, + {"stderr", os.Stderr, true, false}, +} + +func OpenIo(L *LState) int { + mod := L.RegisterModule(IoLibName, map[string]LGFunction{}).(*LTable) + mt := L.NewTypeMetatable(lFileClass) + mt.RawSetString("__index", mt) + L.SetFuncs(mt, fileMethods) + mt.RawSetString("lines", L.NewClosure(fileLines, L.NewFunction(fileLinesIter))) + + for _, finfo := range stdFiles { + file, _ := newFile(L, finfo.file, "", 0, os.FileMode(0), finfo.writable, finfo.readable) + mod.RawSetString(finfo.name, file) + } + uv := L.CreateTable(2, 0) + uv.RawSetInt(fileDefOutIndex, mod.RawGetString("stdout")) + uv.RawSetInt(fileDefInIndex, mod.RawGetString("stdin")) + for name, fn := range ioFuncs { + mod.RawSetString(name, L.NewClosure(fn, uv)) + } + mod.RawSetString("lines", L.NewClosure(ioLines, uv, L.NewClosure(ioLinesIter, uv))) + // Modifications are being made in-place rather than returned? + L.Push(mod) + return 1 +} + +var fileMethods = map[string]LGFunction{ + "__tostring": fileToString, + "write": fileWrite, + "close": fileClose, + "flush": fileFlush, + "lines": fileLines, + "read": fileRead, + "seek": fileSeek, + "setvbuf": fileSetVBuf, +} + +func fileToString(L *LState) int { + file := checkFile(L) + if file.Type() == lFileFile { + if file.closed { + L.Push(LString("file (closed)")) + } else { + L.Push(LString("file")) + } + } else { + if file.closed { + L.Push(LString("process (closed)")) + } else { + L.Push(LString("process")) + } + } + return 1 +} + +func fileWriteAux(L *LState, file *lFile, idx int) int { + if n := fileIsWritable(L, file); n != 0 { + return n + } + errorIfFileIsClosed(L, file) + top := L.GetTop() + out := file.writer + var err error + for i := idx; i <= top; i++ { + L.CheckTypes(i, LTNumber, LTString) + s := LVAsString(L.Get(i)) + if _, err = out.Write(unsafeFastStringToReadOnlyBytes(s)); err != nil { + goto errreturn + } + } + + file.AbandonReadBuffer() + L.Push(LTrue) + return 1 +errreturn: + + file.AbandonReadBuffer() + L.Push(LNil) + L.Push(LString(err.Error())) + L.Push(LNumber(1)) // C-Lua compatibility: Original Lua pushes errno to the stack + return 3 +} + +func fileCloseAux(L *LState, file *lFile) int { + file.closed = true + var err error + if file.writer != nil { + if bwriter, ok := file.writer.(*bufio.Writer); ok { + if err = bwriter.Flush(); err != nil { + goto errreturn + } + } + } + file.AbandonReadBuffer() + + switch file.Type() { + case lFileFile: + if err = file.fp.Close(); err != nil { + goto errreturn + } + L.Push(LTrue) + return 1 + case lFileProcess: + if file.stdout != nil { + file.stdout.Close() // ignore errors + } + err = file.pp.Wait() + var exitStatus int // Initialised to zero value = 0 + if err != nil { + if e2, ok := err.(*exec.ExitError); ok { + if s, ok := e2.Sys().(syscall.WaitStatus); ok { + exitStatus = s.ExitStatus() + } else { + err = errors.New("Unimplemented for system where exec.ExitError.Sys() is not syscall.WaitStatus.") + } + } + } else { + exitStatus = 0 + } + L.Push(LNumber(exitStatus)) + return 1 + } + +errreturn: + L.RaiseError(err.Error()) + return 0 +} + +func fileFlushAux(L *LState, file *lFile) int { + if n := fileIsWritable(L, file); n != 0 { + return n + } + errorIfFileIsClosed(L, file) + + if bwriter, ok := file.writer.(*bufio.Writer); ok { + if err := bwriter.Flush(); err != nil { + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 + } + } + L.Push(LTrue) + return 1 +} + +func fileReadAux(L *LState, file *lFile, idx int) int { + if n := fileIsReadable(L, file); n != 0 { + return n + } + errorIfFileIsClosed(L, file) + if L.GetTop() == idx-1 { + L.Push(LString("*l")) + } + var err error + top := L.GetTop() + for i := idx; i <= top; i++ { + switch lv := L.Get(i).(type) { + case LNumber: + size := int64(lv) + if size == 0 { + _, err = file.reader.ReadByte() + if err == io.EOF { + L.Push(LNil) + goto normalreturn + } + file.reader.UnreadByte() + } + var buf []byte + var iseof bool + buf, err, iseof = readBufioSize(file.reader, size) + if iseof { + L.Push(LNil) + goto normalreturn + } + if err != nil { + goto errreturn + } + L.Push(LString(string(buf))) + case LString: + options := L.CheckString(i) + if len(options) > 0 && options[0] != '*' { + L.ArgError(2, "invalid options:"+options) + } + for _, opt := range options[1:] { + switch opt { + case 'n': + var v LNumber + _, err = fmt.Fscanf(file.reader, LNumberScanFormat, &v) + if err == io.EOF { + L.Push(LNil) + goto normalreturn + } + if err != nil { + goto errreturn + } + L.Push(v) + case 'a': + var buf []byte + buf, err = ioutil.ReadAll(file.reader) + if err == io.EOF { + L.Push(emptyLString) + goto normalreturn + } + if err != nil { + goto errreturn + } + L.Push(LString(string(buf))) + case 'l': + var buf []byte + var iseof bool + buf, err, iseof = readBufioLine(file.reader) + if iseof { + L.Push(LNil) + goto normalreturn + } + if err != nil { + goto errreturn + } + L.Push(LString(string(buf))) + default: + L.ArgError(2, "invalid options:"+string(opt)) + } + } + } + } +normalreturn: + return L.GetTop() - top + +errreturn: + L.RaiseError(err.Error()) + //L.Push(LNil) + //L.Push(LString(err.Error())) + return 2 +} + +var fileSeekOptions = []string{"set", "cur", "end"} + +func fileSeek(L *LState) int { + file := checkFile(L) + if file.Type() != lFileFile { + L.Push(LNil) + L.Push(LString("can not seek a process.")) + return 2 + } + + top := L.GetTop() + if top == 1 { + L.Push(LString("cur")) + L.Push(LNumber(0)) + } else if top == 2 { + L.Push(LNumber(0)) + } + + var pos int64 + var err error + + err = file.AbandonReadBuffer() + if err != nil { + goto errreturn + } + + pos, err = file.fp.Seek(L.CheckInt64(3), L.CheckOption(2, fileSeekOptions)) + if err != nil { + goto errreturn + } + + L.Push(LNumber(pos)) + return 1 + +errreturn: + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 +} + +func fileWrite(L *LState) int { + return fileWriteAux(L, checkFile(L), 2) +} + +func fileClose(L *LState) int { + return fileCloseAux(L, checkFile(L)) +} + +func fileFlush(L *LState) int { + return fileFlushAux(L, checkFile(L)) +} + +func fileLinesIter(L *LState) int { + var file *lFile + if ud, ok := L.Get(1).(*LUserData); ok { + file = ud.Value.(*lFile) + } else { + file = L.Get(UpvalueIndex(2)).(*LUserData).Value.(*lFile) + } + buf, _, err := file.reader.ReadLine() + if err != nil { + if err == io.EOF { + L.Push(LNil) + return 1 + } + L.RaiseError(err.Error()) + } + L.Push(LString(string(buf))) + return 1 +} + +func fileLines(L *LState) int { + file := checkFile(L) + ud := L.CheckUserData(1) + if n := fileIsReadable(L, file); n != 0 { + return 0 + } + L.Push(L.NewClosure(fileLinesIter, L.Get(UpvalueIndex(1)), ud)) + return 1 +} + +func fileRead(L *LState) int { + return fileReadAux(L, checkFile(L), 2) +} + +var filebufOptions = []string{"no", "full"} + +func fileSetVBuf(L *LState) int { + var err error + var writer io.Writer + file := checkFile(L) + if n := fileIsWritable(L, file); n != 0 { + return n + } + switch filebufOptions[L.CheckOption(2, filebufOptions)] { + case "no": + switch file.Type() { + case lFileFile: + file.writer = file.fp + case lFileProcess: + file.writer, err = file.pp.StdinPipe() + if err != nil { + goto errreturn + } + } + case "full", "line": // TODO line buffer not supported + bufsize := L.OptInt(3, fileDefaultWriteBuffer) + switch file.Type() { + case lFileFile: + file.writer = bufio.NewWriterSize(file.fp, bufsize) + case lFileProcess: + writer, err = file.pp.StdinPipe() + if err != nil { + goto errreturn + } + file.writer = bufio.NewWriterSize(writer, bufsize) + } + } + L.Push(LTrue) + return 1 +errreturn: + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 +} + +func ioInput(L *LState) int { + if L.GetTop() == 0 { + L.Push(fileDefIn(L)) + return 1 + } + switch lv := L.Get(1).(type) { + case LString: + file, err := newFile(L, nil, string(lv), os.O_RDONLY, 0600, false, true) + if err != nil { + L.RaiseError(err.Error()) + } + L.Get(UpvalueIndex(1)).(*LTable).RawSetInt(fileDefInIndex, file) + L.Push(file) + return 1 + case *LUserData: + if _, ok := lv.Value.(*lFile); ok { + L.Get(UpvalueIndex(1)).(*LTable).RawSetInt(fileDefInIndex, lv) + L.Push(lv) + return 1 + } + + } + L.ArgError(1, "string or file expedted, but got "+L.Get(1).Type().String()) + return 0 +} + +func ioClose(L *LState) int { + if L.GetTop() == 0 { + return fileCloseAux(L, fileDefOut(L).Value.(*lFile)) + } + return fileClose(L) +} + +func ioFlush(L *LState) int { + return fileFlushAux(L, fileDefOut(L).Value.(*lFile)) +} + +func ioLinesIter(L *LState) int { + var file *lFile + toclose := false + if ud, ok := L.Get(1).(*LUserData); ok { + file = ud.Value.(*lFile) + } else { + file = L.Get(UpvalueIndex(2)).(*LUserData).Value.(*lFile) + toclose = true + } + buf, _, err := file.reader.ReadLine() + if err != nil { + if err == io.EOF { + if toclose { + fileCloseAux(L, file) + } + L.Push(LNil) + return 1 + } + L.RaiseError(err.Error()) + } + L.Push(LString(string(buf))) + return 1 +} + +func ioLines(L *LState) int { + if L.GetTop() == 0 { + L.Push(L.Get(UpvalueIndex(2))) + L.Push(fileDefIn(L)) + return 2 + } + + path := L.CheckString(1) + ud, err := newFile(L, nil, path, os.O_RDONLY, os.FileMode(0600), false, true) + if err != nil { + return 0 + } + L.Push(L.NewClosure(ioLinesIter, L.Get(UpvalueIndex(1)), ud)) + return 1 +} + +var ioOpenOpions = []string{"r", "rb", "w", "wb", "a", "ab", "r+", "rb+", "w+", "wb+", "a+", "ab+"} + +func ioOpenFile(L *LState) int { + path := L.CheckString(1) + if L.GetTop() == 1 { + L.Push(LString("r")) + } + mode := os.O_RDONLY + perm := 0600 + writable := true + readable := true + switch ioOpenOpions[L.CheckOption(2, ioOpenOpions)] { + case "r", "rb": + mode = os.O_RDONLY + writable = false + case "w", "wb": + mode = os.O_WRONLY | os.O_TRUNC | os.O_CREATE + readable = false + case "a", "ab": + mode = os.O_WRONLY | os.O_APPEND | os.O_CREATE + case "r+", "rb+": + mode = os.O_RDWR + case "w+", "wb+": + mode = os.O_RDWR | os.O_TRUNC | os.O_CREATE + case "a+", "ab+": + mode = os.O_APPEND | os.O_RDWR | os.O_CREATE + } + file, err := newFile(L, nil, path, mode, os.FileMode(perm), writable, readable) + if err != nil { + L.Push(LNil) + L.Push(LString(err.Error())) + L.Push(LNumber(1)) // C-Lua compatibility: Original Lua pushes errno to the stack + return 3 + } + L.Push(file) + return 1 + +} + +var ioPopenOptions = []string{"r", "w"} + +func ioPopen(L *LState) int { + cmd := L.CheckString(1) + if L.GetTop() == 1 { + L.Push(LString("r")) + } else if L.GetTop() > 1 && (L.Get(2)).Type() == LTNil { + L.SetTop(1) + L.Push(LString("r")) + } + var file *LUserData + var err error + + switch ioPopenOptions[L.CheckOption(2, ioPopenOptions)] { + case "r": + file, err = newProcess(L, cmd, false, true) + case "w": + file, err = newProcess(L, cmd, true, false) + } + if err != nil { + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 + } + L.Push(file) + return 1 +} + +func ioRead(L *LState) int { + return fileReadAux(L, fileDefIn(L).Value.(*lFile), 1) +} + +func ioType(L *LState) int { + ud, udok := L.Get(1).(*LUserData) + if !udok { + L.Push(LNil) + return 1 + } + file, ok := ud.Value.(*lFile) + if !ok { + L.Push(LNil) + return 1 + } + if file.closed { + L.Push(LString("closed file")) + return 1 + } + L.Push(LString("file")) + return 1 +} + +func ioTmpFile(L *LState) int { + file, err := ioutil.TempFile("", "") + if err != nil { + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 + } + L.G.tempFiles = append(L.G.tempFiles, file) + ud, _ := newFile(L, file, "", 0, os.FileMode(0), true, true) + L.Push(ud) + return 1 +} + +func ioOutput(L *LState) int { + if L.GetTop() == 0 { + L.Push(fileDefOut(L)) + return 1 + } + switch lv := L.Get(1).(type) { + case LString: + file, err := newFile(L, nil, string(lv), os.O_WRONLY|os.O_CREATE, 0600, true, false) + if err != nil { + L.RaiseError(err.Error()) + } + L.Get(UpvalueIndex(1)).(*LTable).RawSetInt(fileDefOutIndex, file) + L.Push(file) + return 1 + case *LUserData: + if _, ok := lv.Value.(*lFile); ok { + L.Get(UpvalueIndex(1)).(*LTable).RawSetInt(fileDefOutIndex, lv) + L.Push(lv) + return 1 + } + + } + L.ArgError(1, "string or file expedted, but got "+L.Get(1).Type().String()) + return 0 +} + +func ioWrite(L *LState) int { + return fileWriteAux(L, fileDefOut(L).Value.(*lFile), 1) +} + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/linit.go b/scraper-go/vendor/github.com/yuin/gopher-lua/linit.go new file mode 100644 index 00000000..cd96d660 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/linit.go @@ -0,0 +1,54 @@ +package lua + +const ( + // BaseLibName is here for consistency; the base functions have no namespace/library. + BaseLibName = "" + // LoadLibName is here for consistency; the loading system has no namespace/library. + LoadLibName = "package" + // TabLibName is the name of the table Library. + TabLibName = "table" + // IoLibName is the name of the io Library. + IoLibName = "io" + // OsLibName is the name of the os Library. + OsLibName = "os" + // StringLibName is the name of the string Library. + StringLibName = "string" + // MathLibName is the name of the math Library. + MathLibName = "math" + // DebugLibName is the name of the debug Library. + DebugLibName = "debug" + // ChannelLibName is the name of the channel Library. + ChannelLibName = "channel" + // CoroutineLibName is the name of the coroutine Library. + CoroutineLibName = "coroutine" +) + +type luaLib struct { + libName string + libFunc LGFunction +} + +var luaLibs = []luaLib{ + luaLib{LoadLibName, OpenPackage}, + luaLib{BaseLibName, OpenBase}, + luaLib{TabLibName, OpenTable}, + luaLib{IoLibName, OpenIo}, + luaLib{OsLibName, OpenOs}, + luaLib{StringLibName, OpenString}, + luaLib{MathLibName, OpenMath}, + luaLib{DebugLibName, OpenDebug}, + luaLib{ChannelLibName, OpenChannel}, + luaLib{CoroutineLibName, OpenCoroutine}, +} + +// OpenLibs loads the built-in libraries. It is equivalent to running OpenLoad, +// then OpenBase, then iterating over the other OpenXXX functions in any order. +func (ls *LState) OpenLibs() { + // NB: Map iteration order in Go is deliberately randomised, so must open Load/Base + // prior to iterating. + for _, lib := range luaLibs { + ls.Push(ls.NewFunction(lib.libFunc)) + ls.Push(LString(lib.libName)) + ls.Call(1, 0) + } +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/loadlib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/loadlib.go new file mode 100644 index 00000000..40ce122b --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/loadlib.go @@ -0,0 +1,128 @@ +package lua + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +/* load lib {{{ */ + +var loLoaders = []LGFunction{loLoaderPreload, loLoaderLua} + +func loGetPath(env string, defpath string) string { + path := os.Getenv(env) + if len(path) == 0 { + path = defpath + } + path = strings.Replace(path, ";;", ";"+defpath+";", -1) + if os.PathSeparator != '/' { + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + panic(err) + } + path = strings.Replace(path, "!", dir, -1) + } + return path +} + +func loFindFile(L *LState, name, pname string) (string, string) { + name = strings.Replace(name, ".", string(os.PathSeparator), -1) + lv := L.GetField(L.GetField(L.Get(EnvironIndex), "package"), pname) + path, ok := lv.(LString) + if !ok { + L.RaiseError("package.%s must be a string", pname) + } + messages := []string{} + for _, pattern := range strings.Split(string(path), ";") { + luapath := strings.Replace(pattern, "?", name, -1) + if _, err := os.Stat(luapath); err == nil { + return luapath, "" + } else { + messages = append(messages, err.Error()) + } + } + return "", strings.Join(messages, "\n\t") +} + +func OpenPackage(L *LState) int { + packagemod := L.RegisterModule(LoadLibName, loFuncs) + + L.SetField(packagemod, "preload", L.NewTable()) + + loaders := L.CreateTable(len(loLoaders), 0) + for i, loader := range loLoaders { + L.RawSetInt(loaders, i+1, L.NewFunction(loader)) + } + L.SetField(packagemod, "loaders", loaders) + L.SetField(L.Get(RegistryIndex), "_LOADERS", loaders) + + loaded := L.NewTable() + L.SetField(packagemod, "loaded", loaded) + L.SetField(L.Get(RegistryIndex), "_LOADED", loaded) + + L.SetField(packagemod, "path", LString(loGetPath(LuaPath, LuaPathDefault))) + L.SetField(packagemod, "cpath", emptyLString) + + L.SetField(packagemod, "config", LString(LuaDirSep+"\n"+LuaPathSep+ + "\n"+LuaPathMark+"\n"+LuaExecDir+"\n"+LuaIgMark+"\n")) + + L.Push(packagemod) + return 1 +} + +var loFuncs = map[string]LGFunction{ + "loadlib": loLoadLib, + "seeall": loSeeAll, +} + +func loLoaderPreload(L *LState) int { + name := L.CheckString(1) + preload := L.GetField(L.GetField(L.Get(EnvironIndex), "package"), "preload") + if _, ok := preload.(*LTable); !ok { + L.RaiseError("package.preload must be a table") + } + lv := L.GetField(preload, name) + if lv == LNil { + L.Push(LString(fmt.Sprintf("no field package.preload['%s']", name))) + return 1 + } + L.Push(lv) + return 1 +} + +func loLoaderLua(L *LState) int { + name := L.CheckString(1) + path, msg := loFindFile(L, name, "path") + if len(path) == 0 { + L.Push(LString(msg)) + return 1 + } + fn, err1 := L.LoadFile(path) + if err1 != nil { + L.RaiseError(err1.Error()) + } + L.Push(fn) + return 1 +} + +func loLoadLib(L *LState) int { + L.RaiseError("loadlib is not supported") + return 0 +} + +func loSeeAll(L *LState) int { + mod := L.CheckTable(1) + mt := L.GetMetatable(mod) + if mt == LNil { + mt = L.CreateTable(0, 1) + L.SetMetatable(mod, mt) + } + L.SetField(mt, "__index", L.Get(GlobalsIndex)) + return 0 +} + +/* }}} */ + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/mathlib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/mathlib.go new file mode 100644 index 00000000..e612f2f0 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/mathlib.go @@ -0,0 +1,231 @@ +package lua + +import ( + "math" + "math/rand" +) + +func OpenMath(L *LState) int { + mod := L.RegisterModule(MathLibName, mathFuncs).(*LTable) + mod.RawSetString("pi", LNumber(math.Pi)) + mod.RawSetString("huge", LNumber(math.MaxFloat64)) + L.Push(mod) + return 1 +} + +var mathFuncs = map[string]LGFunction{ + "abs": mathAbs, + "acos": mathAcos, + "asin": mathAsin, + "atan": mathAtan, + "atan2": mathAtan2, + "ceil": mathCeil, + "cos": mathCos, + "cosh": mathCosh, + "deg": mathDeg, + "exp": mathExp, + "floor": mathFloor, + "fmod": mathFmod, + "frexp": mathFrexp, + "ldexp": mathLdexp, + "log": mathLog, + "log10": mathLog10, + "max": mathMax, + "min": mathMin, + "mod": mathMod, + "modf": mathModf, + "pow": mathPow, + "rad": mathRad, + "random": mathRandom, + "randomseed": mathRandomseed, + "sin": mathSin, + "sinh": mathSinh, + "sqrt": mathSqrt, + "tan": mathTan, + "tanh": mathTanh, +} + +func mathAbs(L *LState) int { + L.Push(LNumber(math.Abs(float64(L.CheckNumber(1))))) + return 1 +} + +func mathAcos(L *LState) int { + L.Push(LNumber(math.Acos(float64(L.CheckNumber(1))))) + return 1 +} + +func mathAsin(L *LState) int { + L.Push(LNumber(math.Asin(float64(L.CheckNumber(1))))) + return 1 +} + +func mathAtan(L *LState) int { + L.Push(LNumber(math.Atan(float64(L.CheckNumber(1))))) + return 1 +} + +func mathAtan2(L *LState) int { + L.Push(LNumber(math.Atan2(float64(L.CheckNumber(1)), float64(L.CheckNumber(2))))) + return 1 +} + +func mathCeil(L *LState) int { + L.Push(LNumber(math.Ceil(float64(L.CheckNumber(1))))) + return 1 +} + +func mathCos(L *LState) int { + L.Push(LNumber(math.Cos(float64(L.CheckNumber(1))))) + return 1 +} + +func mathCosh(L *LState) int { + L.Push(LNumber(math.Cosh(float64(L.CheckNumber(1))))) + return 1 +} + +func mathDeg(L *LState) int { + L.Push(LNumber(float64(L.CheckNumber(1)) * 180 / math.Pi)) + return 1 +} + +func mathExp(L *LState) int { + L.Push(LNumber(math.Exp(float64(L.CheckNumber(1))))) + return 1 +} + +func mathFloor(L *LState) int { + L.Push(LNumber(math.Floor(float64(L.CheckNumber(1))))) + return 1 +} + +func mathFmod(L *LState) int { + L.Push(LNumber(math.Mod(float64(L.CheckNumber(1)), float64(L.CheckNumber(2))))) + return 1 +} + +func mathFrexp(L *LState) int { + v1, v2 := math.Frexp(float64(L.CheckNumber(1))) + L.Push(LNumber(v1)) + L.Push(LNumber(v2)) + return 2 +} + +func mathLdexp(L *LState) int { + L.Push(LNumber(math.Ldexp(float64(L.CheckNumber(1)), L.CheckInt(2)))) + return 1 +} + +func mathLog(L *LState) int { + L.Push(LNumber(math.Log(float64(L.CheckNumber(1))))) + return 1 +} + +func mathLog10(L *LState) int { + L.Push(LNumber(math.Log10(float64(L.CheckNumber(1))))) + return 1 +} + +func mathMax(L *LState) int { + if L.GetTop() == 0 { + L.RaiseError("wrong number of arguments") + } + max := L.CheckNumber(1) + top := L.GetTop() + for i := 2; i <= top; i++ { + v := L.CheckNumber(i) + if v > max { + max = v + } + } + L.Push(max) + return 1 +} + +func mathMin(L *LState) int { + if L.GetTop() == 0 { + L.RaiseError("wrong number of arguments") + } + min := L.CheckNumber(1) + top := L.GetTop() + for i := 2; i <= top; i++ { + v := L.CheckNumber(i) + if v < min { + min = v + } + } + L.Push(min) + return 1 +} + +func mathMod(L *LState) int { + lhs := L.CheckNumber(1) + rhs := L.CheckNumber(2) + L.Push(luaModulo(lhs, rhs)) + return 1 +} + +func mathModf(L *LState) int { + v1, v2 := math.Modf(float64(L.CheckNumber(1))) + L.Push(LNumber(v1)) + L.Push(LNumber(v2)) + return 2 +} + +func mathPow(L *LState) int { + L.Push(LNumber(math.Pow(float64(L.CheckNumber(1)), float64(L.CheckNumber(2))))) + return 1 +} + +func mathRad(L *LState) int { + L.Push(LNumber(float64(L.CheckNumber(1)) * math.Pi / 180)) + return 1 +} + +func mathRandom(L *LState) int { + switch L.GetTop() { + case 0: + L.Push(LNumber(rand.Float64())) + case 1: + n := L.CheckInt(1) + L.Push(LNumber(rand.Intn(n) + 1)) + default: + min := L.CheckInt(1) + max := L.CheckInt(2) + 1 + L.Push(LNumber(rand.Intn(max-min) + min)) + } + return 1 +} + +func mathRandomseed(L *LState) int { + rand.Seed(L.CheckInt64(1)) + return 0 +} + +func mathSin(L *LState) int { + L.Push(LNumber(math.Sin(float64(L.CheckNumber(1))))) + return 1 +} + +func mathSinh(L *LState) int { + L.Push(LNumber(math.Sinh(float64(L.CheckNumber(1))))) + return 1 +} + +func mathSqrt(L *LState) int { + L.Push(LNumber(math.Sqrt(float64(L.CheckNumber(1))))) + return 1 +} + +func mathTan(L *LState) int { + L.Push(LNumber(math.Tan(float64(L.CheckNumber(1))))) + return 1 +} + +func mathTanh(L *LState) int { + L.Push(LNumber(math.Tanh(float64(L.CheckNumber(1))))) + return 1 +} + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/opcode.go b/scraper-go/vendor/github.com/yuin/gopher-lua/opcode.go new file mode 100644 index 00000000..91fff1c9 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/opcode.go @@ -0,0 +1,371 @@ +package lua + +import ( + "fmt" +) + +/* + gopherlua uses Lua 5.1.4's opcodes. + Lua 5.1.4 opcodes layout: + + instruction = 32bit(fixed length) + + +---------------------------------------------+ + |0-5(6bits)|6-13(8bit)|14-22(9bit)|23-31(9bit)| + |==========+==========+===========+===========| + | opcode | A | C | B | + |----------+----------+-----------+-----------| + | opcode | A | Bx(unsigned) | + |----------+----------+-----------+-----------| + | opcode | A | sBx(signed) | + +---------------------------------------------+ +*/ + +const opInvalidInstruction = ^uint32(0) + +const opSizeCode = 6 +const opSizeA = 8 +const opSizeB = 9 +const opSizeC = 9 +const opSizeBx = 18 +const opSizesBx = 18 + +const opMaxArgsA = (1 << opSizeA) - 1 +const opMaxArgsB = (1 << opSizeB) - 1 +const opMaxArgsC = (1 << opSizeC) - 1 +const opMaxArgBx = (1 << opSizeBx) - 1 +const opMaxArgSbx = opMaxArgBx >> 1 + +const ( + OP_MOVE int = iota /* A B R(A) := R(B) */ + OP_MOVEN /* A B R(A) := R(B); followed by R(C) MOVE ops */ + OP_LOADK /* A Bx R(A) := Kst(Bx) */ + OP_LOADBOOL /* A B C R(A) := (Bool)B; if (C) pc++ */ + OP_LOADNIL /* A B R(A) := ... := R(B) := nil */ + OP_GETUPVAL /* A B R(A) := UpValue[B] */ + + OP_GETGLOBAL /* A Bx R(A) := Gbl[Kst(Bx)] */ + OP_GETTABLE /* A B C R(A) := R(B)[RK(C)] */ + OP_GETTABLEKS /* A B C R(A) := R(B)[RK(C)] ; RK(C) is constant string */ + + OP_SETGLOBAL /* A Bx Gbl[Kst(Bx)] := R(A) */ + OP_SETUPVAL /* A B UpValue[B] := R(A) */ + OP_SETTABLE /* A B C R(A)[RK(B)] := RK(C) */ + OP_SETTABLEKS /* A B C R(A)[RK(B)] := RK(C) ; RK(B) is constant string */ + + OP_NEWTABLE /* A B C R(A) := {} (size = BC) */ + + OP_SELF /* A B C R(A+1) := R(B); R(A) := R(B)[RK(C)] */ + + OP_ADD /* A B C R(A) := RK(B) + RK(C) */ + OP_SUB /* A B C R(A) := RK(B) - RK(C) */ + OP_MUL /* A B C R(A) := RK(B) * RK(C) */ + OP_DIV /* A B C R(A) := RK(B) / RK(C) */ + OP_MOD /* A B C R(A) := RK(B) % RK(C) */ + OP_POW /* A B C R(A) := RK(B) ^ RK(C) */ + OP_UNM /* A B R(A) := -R(B) */ + OP_NOT /* A B R(A) := not R(B) */ + OP_LEN /* A B R(A) := length of R(B) */ + + OP_CONCAT /* A B C R(A) := R(B).. ... ..R(C) */ + + OP_JMP /* sBx pc+=sBx */ + + OP_EQ /* A B C if ((RK(B) == RK(C)) ~= A) then pc++ */ + OP_LT /* A B C if ((RK(B) < RK(C)) ~= A) then pc++ */ + OP_LE /* A B C if ((RK(B) <= RK(C)) ~= A) then pc++ */ + + OP_TEST /* A C if not (R(A) <=> C) then pc++ */ + OP_TESTSET /* A B C if (R(B) <=> C) then R(A) := R(B) else pc++ */ + + OP_CALL /* A B C R(A) ... R(A+C-2) := R(A)(R(A+1) ... R(A+B-1)) */ + OP_TAILCALL /* A B C return R(A)(R(A+1) ... R(A+B-1)) */ + OP_RETURN /* A B return R(A) ... R(A+B-2) (see note) */ + + OP_FORLOOP /* A sBx R(A)+=R(A+2); + if R(A) =) R(A)*/ + OP_CLOSURE /* A Bx R(A) := closure(KPROTO[Bx] R(A) ... R(A+n)) */ + + OP_VARARG /* A B R(A) R(A+1) ... R(A+B-1) = vararg */ + + OP_NOP /* NOP */ +) +const opCodeMax = OP_NOP + +type opArgMode int + +const ( + opArgModeN opArgMode = iota + opArgModeU + opArgModeR + opArgModeK +) + +type opType int + +const ( + opTypeABC = iota + opTypeABx + opTypeASbx +) + +type opProp struct { + Name string + IsTest bool + SetRegA bool + ModeArgB opArgMode + ModeArgC opArgMode + Type opType +} + +var opProps = []opProp{ + opProp{"MOVE", false, true, opArgModeR, opArgModeN, opTypeABC}, + opProp{"MOVEN", false, true, opArgModeR, opArgModeN, opTypeABC}, + opProp{"LOADK", false, true, opArgModeK, opArgModeN, opTypeABx}, + opProp{"LOADBOOL", false, true, opArgModeU, opArgModeU, opTypeABC}, + opProp{"LOADNIL", false, true, opArgModeR, opArgModeN, opTypeABC}, + opProp{"GETUPVAL", false, true, opArgModeU, opArgModeN, opTypeABC}, + opProp{"GETGLOBAL", false, true, opArgModeK, opArgModeN, opTypeABx}, + opProp{"GETTABLE", false, true, opArgModeR, opArgModeK, opTypeABC}, + opProp{"GETTABLEKS", false, true, opArgModeR, opArgModeK, opTypeABC}, + opProp{"SETGLOBAL", false, false, opArgModeK, opArgModeN, opTypeABx}, + opProp{"SETUPVAL", false, false, opArgModeU, opArgModeN, opTypeABC}, + opProp{"SETTABLE", false, false, opArgModeK, opArgModeK, opTypeABC}, + opProp{"SETTABLEKS", false, false, opArgModeK, opArgModeK, opTypeABC}, + opProp{"NEWTABLE", false, true, opArgModeU, opArgModeU, opTypeABC}, + opProp{"SELF", false, true, opArgModeR, opArgModeK, opTypeABC}, + opProp{"ADD", false, true, opArgModeK, opArgModeK, opTypeABC}, + opProp{"SUB", false, true, opArgModeK, opArgModeK, opTypeABC}, + opProp{"MUL", false, true, opArgModeK, opArgModeK, opTypeABC}, + opProp{"DIV", false, true, opArgModeK, opArgModeK, opTypeABC}, + opProp{"MOD", false, true, opArgModeK, opArgModeK, opTypeABC}, + opProp{"POW", false, true, opArgModeK, opArgModeK, opTypeABC}, + opProp{"UNM", false, true, opArgModeR, opArgModeN, opTypeABC}, + opProp{"NOT", false, true, opArgModeR, opArgModeN, opTypeABC}, + opProp{"LEN", false, true, opArgModeR, opArgModeN, opTypeABC}, + opProp{"CONCAT", false, true, opArgModeR, opArgModeR, opTypeABC}, + opProp{"JMP", false, false, opArgModeR, opArgModeN, opTypeASbx}, + opProp{"EQ", true, false, opArgModeK, opArgModeK, opTypeABC}, + opProp{"LT", true, false, opArgModeK, opArgModeK, opTypeABC}, + opProp{"LE", true, false, opArgModeK, opArgModeK, opTypeABC}, + opProp{"TEST", true, true, opArgModeR, opArgModeU, opTypeABC}, + opProp{"TESTSET", true, true, opArgModeR, opArgModeU, opTypeABC}, + opProp{"CALL", false, true, opArgModeU, opArgModeU, opTypeABC}, + opProp{"TAILCALL", false, true, opArgModeU, opArgModeU, opTypeABC}, + opProp{"RETURN", false, false, opArgModeU, opArgModeN, opTypeABC}, + opProp{"FORLOOP", false, true, opArgModeR, opArgModeN, opTypeASbx}, + opProp{"FORPREP", false, true, opArgModeR, opArgModeN, opTypeASbx}, + opProp{"TFORLOOP", true, false, opArgModeN, opArgModeU, opTypeABC}, + opProp{"SETLIST", false, false, opArgModeU, opArgModeU, opTypeABC}, + opProp{"CLOSE", false, false, opArgModeN, opArgModeN, opTypeABC}, + opProp{"CLOSURE", false, true, opArgModeU, opArgModeN, opTypeABx}, + opProp{"VARARG", false, true, opArgModeU, opArgModeN, opTypeABC}, + opProp{"NOP", false, false, opArgModeR, opArgModeN, opTypeASbx}, +} + +func opGetOpCode(inst uint32) int { + return int(inst >> 26) +} + +func opSetOpCode(inst *uint32, opcode int) { + *inst = (*inst & 0x3ffffff) | uint32(opcode<<26) +} + +func opGetArgA(inst uint32) int { + return int(inst>>18) & 0xff +} + +func opSetArgA(inst *uint32, arg int) { + *inst = (*inst & 0xfc03ffff) | uint32((arg&0xff)<<18) +} + +func opGetArgB(inst uint32) int { + return int(inst & 0x1ff) +} + +func opSetArgB(inst *uint32, arg int) { + *inst = (*inst & 0xfffffe00) | uint32(arg&0x1ff) +} + +func opGetArgC(inst uint32) int { + return int(inst>>9) & 0x1ff +} + +func opSetArgC(inst *uint32, arg int) { + *inst = (*inst & 0xfffc01ff) | uint32((arg&0x1ff)<<9) +} + +func opGetArgBx(inst uint32) int { + return int(inst & 0x3ffff) +} + +func opSetArgBx(inst *uint32, arg int) { + *inst = (*inst & 0xfffc0000) | uint32(arg&0x3ffff) +} + +func opGetArgSbx(inst uint32) int { + return opGetArgBx(inst) - opMaxArgSbx +} + +func opSetArgSbx(inst *uint32, arg int) { + opSetArgBx(inst, arg+opMaxArgSbx) +} + +func opCreateABC(op int, a int, b int, c int) uint32 { + var inst uint32 = 0 + opSetOpCode(&inst, op) + opSetArgA(&inst, a) + opSetArgB(&inst, b) + opSetArgC(&inst, c) + return inst +} + +func opCreateABx(op int, a int, bx int) uint32 { + var inst uint32 = 0 + opSetOpCode(&inst, op) + opSetArgA(&inst, a) + opSetArgBx(&inst, bx) + return inst +} + +func opCreateASbx(op int, a int, sbx int) uint32 { + var inst uint32 = 0 + opSetOpCode(&inst, op) + opSetArgA(&inst, a) + opSetArgSbx(&inst, sbx) + return inst +} + +const opBitRk = 1 << (opSizeB - 1) +const opMaxIndexRk = opBitRk - 1 + +func opIsK(value int) bool { + return bool((value & opBitRk) != 0) +} + +func opIndexK(value int) int { + return value & ^opBitRk +} + +func opRkAsk(value int) int { + return value | opBitRk +} + +func opToString(inst uint32) string { + op := opGetOpCode(inst) + if op > opCodeMax { + return "" + } + prop := &(opProps[op]) + + arga := opGetArgA(inst) + argb := opGetArgB(inst) + argc := opGetArgC(inst) + argbx := opGetArgBx(inst) + argsbx := opGetArgSbx(inst) + + buf := "" + switch prop.Type { + case opTypeABC: + buf = fmt.Sprintf("%s | %d, %d, %d", prop.Name, arga, argb, argc) + case opTypeABx: + buf = fmt.Sprintf("%s | %d, %d", prop.Name, arga, argbx) + case opTypeASbx: + buf = fmt.Sprintf("%s | %d, %d", prop.Name, arga, argsbx) + } + + switch op { + case OP_MOVE: + buf += fmt.Sprintf("; R(%v) := R(%v)", arga, argb) + case OP_MOVEN: + buf += fmt.Sprintf("; R(%v) := R(%v); followed by %v MOVE ops", arga, argb, argc) + case OP_LOADK: + buf += fmt.Sprintf("; R(%v) := Kst(%v)", arga, argbx) + case OP_LOADBOOL: + buf += fmt.Sprintf("; R(%v) := (Bool)%v; if (%v) pc++", arga, argb, argc) + case OP_LOADNIL: + buf += fmt.Sprintf("; R(%v) := ... := R(%v) := nil", arga, argb) + case OP_GETUPVAL: + buf += fmt.Sprintf("; R(%v) := UpValue[%v]", arga, argb) + case OP_GETGLOBAL: + buf += fmt.Sprintf("; R(%v) := Gbl[Kst(%v)]", arga, argbx) + case OP_GETTABLE: + buf += fmt.Sprintf("; R(%v) := R(%v)[RK(%v)]", arga, argb, argc) + case OP_GETTABLEKS: + buf += fmt.Sprintf("; R(%v) := R(%v)[RK(%v)] ; RK(%v) is constant string", arga, argb, argc, argc) + case OP_SETGLOBAL: + buf += fmt.Sprintf("; Gbl[Kst(%v)] := R(%v)", argbx, arga) + case OP_SETUPVAL: + buf += fmt.Sprintf("; UpValue[%v] := R(%v)", argb, arga) + case OP_SETTABLE: + buf += fmt.Sprintf("; R(%v)[RK(%v)] := RK(%v)", arga, argb, argc) + case OP_SETTABLEKS: + buf += fmt.Sprintf("; R(%v)[RK(%v)] := RK(%v) ; RK(%v) is constant string", arga, argb, argc, argb) + case OP_NEWTABLE: + buf += fmt.Sprintf("; R(%v) := {} (size = BC)", arga) + case OP_SELF: + buf += fmt.Sprintf("; R(%v+1) := R(%v); R(%v) := R(%v)[RK(%v)]", arga, argb, arga, argb, argc) + case OP_ADD: + buf += fmt.Sprintf("; R(%v) := RK(%v) + RK(%v)", arga, argb, argc) + case OP_SUB: + buf += fmt.Sprintf("; R(%v) := RK(%v) - RK(%v)", arga, argb, argc) + case OP_MUL: + buf += fmt.Sprintf("; R(%v) := RK(%v) * RK(%v)", arga, argb, argc) + case OP_DIV: + buf += fmt.Sprintf("; R(%v) := RK(%v) / RK(%v)", arga, argb, argc) + case OP_MOD: + buf += fmt.Sprintf("; R(%v) := RK(%v) %% RK(%v)", arga, argb, argc) + case OP_POW: + buf += fmt.Sprintf("; R(%v) := RK(%v) ^ RK(%v)", arga, argb, argc) + case OP_UNM: + buf += fmt.Sprintf("; R(%v) := -R(%v)", arga, argb) + case OP_NOT: + buf += fmt.Sprintf("; R(%v) := not R(%v)", arga, argb) + case OP_LEN: + buf += fmt.Sprintf("; R(%v) := length of R(%v)", arga, argb) + case OP_CONCAT: + buf += fmt.Sprintf("; R(%v) := R(%v).. ... ..R(%v)", arga, argb, argc) + case OP_JMP: + buf += fmt.Sprintf("; pc+=%v", argsbx) + case OP_EQ: + buf += fmt.Sprintf("; if ((RK(%v) == RK(%v)) ~= %v) then pc++", argb, argc, arga) + case OP_LT: + buf += fmt.Sprintf("; if ((RK(%v) < RK(%v)) ~= %v) then pc++", argb, argc, arga) + case OP_LE: + buf += fmt.Sprintf("; if ((RK(%v) <= RK(%v)) ~= %v) then pc++", argb, argc, arga) + case OP_TEST: + buf += fmt.Sprintf("; if not (R(%v) <=> %v) then pc++", arga, argc) + case OP_TESTSET: + buf += fmt.Sprintf("; if (R(%v) <=> %v) then R(%v) := R(%v) else pc++", argb, argc, arga, argb) + case OP_CALL: + buf += fmt.Sprintf("; R(%v) ... R(%v+%v-2) := R(%v)(R(%v+1) ... R(%v+%v-1))", arga, arga, argc, arga, arga, arga, argb) + case OP_TAILCALL: + buf += fmt.Sprintf("; return R(%v)(R(%v+1) ... R(%v+%v-1))", arga, arga, arga, argb) + case OP_RETURN: + buf += fmt.Sprintf("; return R(%v) ... R(%v+%v-2)", arga, arga, argb) + case OP_FORLOOP: + buf += fmt.Sprintf("; R(%v)+=R(%v+2); if R(%v) =) R(%v)", arga) + case OP_CLOSURE: + buf += fmt.Sprintf("; R(%v) := closure(KPROTO[%v] R(%v) ... R(%v+n))", arga, argbx, arga, arga) + case OP_VARARG: + buf += fmt.Sprintf("; R(%v) R(%v+1) ... R(%v+%v-1) = vararg", arga, arga, arga, argb) + case OP_NOP: + /* nothing to do */ + } + return buf +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/oslib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/oslib.go new file mode 100644 index 00000000..5448cc1f --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/oslib.go @@ -0,0 +1,236 @@ +package lua + +import ( + "io/ioutil" + "os" + "strings" + "time" +) + +var startedAt time.Time + +func init() { + startedAt = time.Now() +} + +func getIntField(L *LState, tb *LTable, key string, v int) int { + ret := tb.RawGetString(key) + + switch lv := ret.(type) { + case LNumber: + return int(lv) + case LString: + slv := string(lv) + slv = strings.TrimLeft(slv, " ") + if strings.HasPrefix(slv, "0") && !strings.HasPrefix(slv, "0x") && !strings.HasPrefix(slv, "0X") { + // Standard lua interpreter only support decimal and hexadecimal + slv = strings.TrimLeft(slv, "0") + if slv == "" { + return 0 + } + } + if num, err := parseNumber(slv); err == nil { + return int(num) + } + default: + return v + } + + return v +} + +func getBoolField(L *LState, tb *LTable, key string, v bool) bool { + ret := tb.RawGetString(key) + if lb, ok := ret.(LBool); ok { + return bool(lb) + } + return v +} + +func OpenOs(L *LState) int { + osmod := L.RegisterModule(OsLibName, osFuncs) + L.Push(osmod) + return 1 +} + +var osFuncs = map[string]LGFunction{ + "clock": osClock, + "difftime": osDiffTime, + "execute": osExecute, + "exit": osExit, + "date": osDate, + "getenv": osGetEnv, + "remove": osRemove, + "rename": osRename, + "setenv": osSetEnv, + "setlocale": osSetLocale, + "time": osTime, + "tmpname": osTmpname, +} + +func osClock(L *LState) int { + L.Push(LNumber(float64(time.Now().Sub(startedAt)) / float64(time.Second))) + return 1 +} + +func osDiffTime(L *LState) int { + L.Push(LNumber(L.CheckInt64(1) - L.CheckInt64(2))) + return 1 +} + +func osExecute(L *LState) int { + var procAttr os.ProcAttr + procAttr.Files = []*os.File{os.Stdin, os.Stdout, os.Stderr} + cmd, args := popenArgs(L.CheckString(1)) + args = append([]string{cmd}, args...) + process, err := os.StartProcess(cmd, args, &procAttr) + if err != nil { + L.Push(LNumber(1)) + return 1 + } + + ps, err := process.Wait() + if err != nil || !ps.Success() { + L.Push(LNumber(1)) + return 1 + } + L.Push(LNumber(0)) + return 1 +} + +func osExit(L *LState) int { + L.Close() + os.Exit(L.OptInt(1, 0)) + return 1 +} + +func osDate(L *LState) int { + t := time.Now() + isUTC := false + cfmt := "%c" + if L.GetTop() >= 1 { + cfmt = L.CheckString(1) + if strings.HasPrefix(cfmt, "!") { + cfmt = strings.TrimLeft(cfmt, "!") + isUTC = true + } + if L.GetTop() >= 2 { + t = time.Unix(L.CheckInt64(2), 0) + } + if isUTC { + t = t.UTC() + } + if strings.HasPrefix(cfmt, "*t") { + ret := L.NewTable() + ret.RawSetString("year", LNumber(t.Year())) + ret.RawSetString("month", LNumber(t.Month())) + ret.RawSetString("day", LNumber(t.Day())) + ret.RawSetString("hour", LNumber(t.Hour())) + ret.RawSetString("min", LNumber(t.Minute())) + ret.RawSetString("sec", LNumber(t.Second())) + ret.RawSetString("wday", LNumber(t.Weekday()+1)) + // TODO yday & dst + ret.RawSetString("yday", LNumber(0)) + ret.RawSetString("isdst", LFalse) + L.Push(ret) + return 1 + } + } + L.Push(LString(strftime(t, cfmt))) + return 1 +} + +func osGetEnv(L *LState) int { + v := os.Getenv(L.CheckString(1)) + if len(v) == 0 { + L.Push(LNil) + } else { + L.Push(LString(v)) + } + return 1 +} + +func osRemove(L *LState) int { + err := os.Remove(L.CheckString(1)) + if err != nil { + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 + } else { + L.Push(LTrue) + return 1 + } +} + +func osRename(L *LState) int { + err := os.Rename(L.CheckString(1), L.CheckString(2)) + if err != nil { + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 + } else { + L.Push(LTrue) + return 1 + } +} + +func osSetLocale(L *LState) int { + // setlocale is not supported + L.Push(LFalse) + return 1 +} + +func osSetEnv(L *LState) int { + err := os.Setenv(L.CheckString(1), L.CheckString(2)) + if err != nil { + L.Push(LNil) + L.Push(LString(err.Error())) + return 2 + } else { + L.Push(LTrue) + return 1 + } +} + +func osTime(L *LState) int { + if L.GetTop() == 0 { + L.Push(LNumber(time.Now().Unix())) + } else { + lv := L.CheckAny(1) + if lv == LNil { + L.Push(LNumber(time.Now().Unix())) + } else { + tbl, ok := lv.(*LTable) + if !ok { + L.TypeError(1, LTTable) + } + sec := getIntField(L, tbl, "sec", 0) + min := getIntField(L, tbl, "min", 0) + hour := getIntField(L, tbl, "hour", 12) + day := getIntField(L, tbl, "day", -1) + month := getIntField(L, tbl, "month", -1) + year := getIntField(L, tbl, "year", -1) + isdst := getBoolField(L, tbl, "isdst", false) + t := time.Date(year, time.Month(month), day, hour, min, sec, 0, time.Local) + // TODO dst + if false { + print(isdst) + } + L.Push(LNumber(t.Unix())) + } + } + return 1 +} + +func osTmpname(L *LState) int { + file, err := ioutil.TempFile("", "") + if err != nil { + L.RaiseError("unable to generate a unique filename") + } + file.Close() + os.Remove(file.Name()) // ignore errors + L.Push(LString(file.Name())) + return 1 +} + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/package.go b/scraper-go/vendor/github.com/yuin/gopher-lua/package.go new file mode 100644 index 00000000..9fde3f0c --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/package.go @@ -0,0 +1,7 @@ +// GopherLua: VM and compiler for Lua in Go +package lua + +const PackageName = "GopherLua" +const PackageVersion = "0.1" +const PackageAuthors = "Yusuke Inuzuka" +const PackageCopyRight = PackageName + " " + PackageVersion + " Copyright (C) 2015 -2017 " + PackageAuthors diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/parse/Makefile b/scraper-go/vendor/github.com/yuin/gopher-lua/parse/Makefile new file mode 100644 index 00000000..9838b139 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/parse/Makefile @@ -0,0 +1,7 @@ +all : parser.go + +parser.go : parser.go.y + goyacc -o $@ parser.go.y; [ -f y.output ] && ( rm -f y.output ) + +clean: + rm -f parser.go diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/parse/lexer.go b/scraper-go/vendor/github.com/yuin/gopher-lua/parse/lexer.go new file mode 100644 index 00000000..c20a0bdd --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/parse/lexer.go @@ -0,0 +1,549 @@ +package parse + +import ( + "bufio" + "bytes" + "fmt" + "io" + "reflect" + "strconv" + "strings" + + "github.com/yuin/gopher-lua/ast" +) + +const EOF = -1 +const whitespace1 = 1<<'\t' | 1<<' ' +const whitespace2 = 1<<'\t' | 1<<'\n' | 1<<'\r' | 1<<' ' + +type Error struct { + Pos ast.Position + Message string + Token string +} + +func (e *Error) Error() string { + pos := e.Pos + if pos.Line == EOF { + return fmt.Sprintf("%v at EOF: %s\n", pos.Source, e.Message) + } else { + return fmt.Sprintf("%v line:%d(column:%d) near '%v': %s\n", pos.Source, pos.Line, pos.Column, e.Token, e.Message) + } +} + +func writeChar(buf *bytes.Buffer, c int) { buf.WriteByte(byte(c)) } + +func isDecimal(ch int) bool { return '0' <= ch && ch <= '9' } + +func isIdent(ch int, pos int) bool { + return ch == '_' || 'A' <= ch && ch <= 'Z' || 'a' <= ch && ch <= 'z' || isDecimal(ch) && pos > 0 +} + +func isDigit(ch int) bool { + return '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F' +} + +type Scanner struct { + Pos ast.Position + reader *bufio.Reader +} + +func NewScanner(reader io.Reader, source string) *Scanner { + return &Scanner{ + Pos: ast.Position{ + Source: source, + Line: 1, + Column: 0, + }, + reader: bufio.NewReaderSize(reader, 4096), + } +} + +func (sc *Scanner) Error(tok string, msg string) *Error { return &Error{sc.Pos, msg, tok} } + +func (sc *Scanner) TokenError(tok ast.Token, msg string) *Error { return &Error{tok.Pos, msg, tok.Str} } + +func (sc *Scanner) readNext() int { + ch, err := sc.reader.ReadByte() + if err == io.EOF { + return EOF + } + return int(ch) +} + +func (sc *Scanner) Newline(ch int) { + if ch < 0 { + return + } + sc.Pos.Line += 1 + sc.Pos.Column = 0 + next := sc.Peek() + if ch == '\n' && next == '\r' || ch == '\r' && next == '\n' { + sc.reader.ReadByte() + } +} + +func (sc *Scanner) Next() int { + ch := sc.readNext() + switch ch { + case '\n', '\r': + sc.Newline(ch) + ch = int('\n') + case EOF: + sc.Pos.Line = EOF + sc.Pos.Column = 0 + default: + sc.Pos.Column++ + } + return ch +} + +func (sc *Scanner) Peek() int { + ch := sc.readNext() + if ch != EOF { + sc.reader.UnreadByte() + } + return ch +} + +func (sc *Scanner) skipWhiteSpace(whitespace int64) int { + ch := sc.Next() + for ; whitespace&(1<': + if sc.Peek() == '=' { + tok.Type = TGte + tok.Str = ">=" + sc.Next() + } else { + tok.Type = ch + tok.Str = string(rune(ch)) + } + case '.': + ch2 := sc.Peek() + switch { + case isDecimal(ch2): + tok.Type = TNumber + err = sc.scanNumber(ch, buf) + tok.Str = buf.String() + case ch2 == '.': + writeChar(buf, ch) + writeChar(buf, sc.Next()) + if sc.Peek() == '.' { + writeChar(buf, sc.Next()) + tok.Type = T3Comma + } else { + tok.Type = T2Comma + } + default: + tok.Type = '.' + } + tok.Str = buf.String() + case ':': + if sc.Peek() == ':' { + tok.Type = T2Colon + tok.Str = "::" + sc.Next() + } else { + tok.Type = ch + tok.Str = string(rune(ch)) + } + case '+', '*', '/', '%', '^', '#', '(', ')', '{', '}', ']', ';', ',': + tok.Type = ch + tok.Str = string(rune(ch)) + default: + writeChar(buf, ch) + err = sc.Error(buf.String(), "Invalid token") + goto finally + } + } + +finally: + tok.Name = TokenName(int(tok.Type)) + return tok, err +} + +// yacc interface {{{ + +type Lexer struct { + scanner *Scanner + Stmts []ast.Stmt + PNewLine bool + Token ast.Token + PrevTokenType int +} + +func (lx *Lexer) Lex(lval *yySymType) int { + lx.PrevTokenType = lx.Token.Type + tok, err := lx.scanner.Scan(lx) + if err != nil { + panic(err) + } + if tok.Type < 0 { + return 0 + } + lval.token = tok + lx.Token = tok + return int(tok.Type) +} + +func (lx *Lexer) Error(message string) { + panic(lx.scanner.Error(lx.Token.Str, message)) +} + +func (lx *Lexer) TokenError(tok ast.Token, message string) { + panic(lx.scanner.TokenError(tok, message)) +} + +func Parse(reader io.Reader, name string) (chunk []ast.Stmt, err error) { + lexer := &Lexer{NewScanner(reader, name), nil, false, ast.Token{Str: ""}, TNil} + chunk = nil + defer func() { + if e := recover(); e != nil { + err, _ = e.(error) + } + }() + yyParse(lexer) + chunk = lexer.Stmts + return +} + +// }}} + +// Dump {{{ + +func isInlineDumpNode(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Struct, reflect.Slice, reflect.Interface, reflect.Ptr: + return false + default: + return true + } +} + +func dump(node interface{}, level int, s string) string { + rt := reflect.TypeOf(node) + if fmt.Sprint(rt) == "" { + return strings.Repeat(s, level) + "" + } + + rv := reflect.ValueOf(node) + buf := []string{} + switch rt.Kind() { + case reflect.Slice: + if rv.Len() == 0 { + return strings.Repeat(s, level) + "" + } + for i := 0; i < rv.Len(); i++ { + buf = append(buf, dump(rv.Index(i).Interface(), level, s)) + } + case reflect.Ptr: + vt := rv.Elem() + tt := rt.Elem() + indicies := []int{} + for i := 0; i < tt.NumField(); i++ { + if strings.Index(tt.Field(i).Name, "Base") > -1 { + continue + } + indicies = append(indicies, i) + } + switch { + case len(indicies) == 0: + return strings.Repeat(s, level) + "" + case len(indicies) == 1 && isInlineDumpNode(vt.Field(indicies[0])): + for _, i := range indicies { + buf = append(buf, strings.Repeat(s, level)+"- Node$"+tt.Name()+": "+dump(vt.Field(i).Interface(), 0, s)) + } + default: + buf = append(buf, strings.Repeat(s, level)+"- Node$"+tt.Name()) + for _, i := range indicies { + if isInlineDumpNode(vt.Field(i)) { + inf := dump(vt.Field(i).Interface(), 0, s) + buf = append(buf, strings.Repeat(s, level+1)+tt.Field(i).Name+": "+inf) + } else { + buf = append(buf, strings.Repeat(s, level+1)+tt.Field(i).Name+": ") + buf = append(buf, dump(vt.Field(i).Interface(), level+2, s)) + } + } + } + default: + buf = append(buf, strings.Repeat(s, level)+fmt.Sprint(node)) + } + return strings.Join(buf, "\n") +} + +func Dump(chunk []ast.Stmt) string { + return dump(chunk, 0, " ") +} + +// }} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/parse/parser.go b/scraper-go/vendor/github.com/yuin/gopher-lua/parse/parser.go new file mode 100644 index 00000000..aee06c8c --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/parse/parser.go @@ -0,0 +1,1385 @@ +// Code generated by goyacc -o parser.go parser.go.y. DO NOT EDIT. + +//line parser.go.y:2 +package parse + +import __yyfmt__ "fmt" + +//line parser.go.y:2 + +import ( + "github.com/yuin/gopher-lua/ast" +) + +//line parser.go.y:34 +type yySymType struct { + yys int + token ast.Token + + stmts []ast.Stmt + stmt ast.Stmt + + funcname *ast.FuncName + funcexpr *ast.FunctionExpr + + exprlist []ast.Expr + expr ast.Expr + + fieldlist []*ast.Field + field *ast.Field + fieldsep string + + namelist []string + parlist *ast.ParList +} + +const TAnd = 57346 +const TBreak = 57347 +const TDo = 57348 +const TElse = 57349 +const TElseIf = 57350 +const TEnd = 57351 +const TFalse = 57352 +const TFor = 57353 +const TFunction = 57354 +const TIf = 57355 +const TIn = 57356 +const TLocal = 57357 +const TNil = 57358 +const TNot = 57359 +const TOr = 57360 +const TReturn = 57361 +const TRepeat = 57362 +const TThen = 57363 +const TTrue = 57364 +const TUntil = 57365 +const TWhile = 57366 +const TGoto = 57367 +const TEqeq = 57368 +const TNeq = 57369 +const TLte = 57370 +const TGte = 57371 +const T2Comma = 57372 +const T3Comma = 57373 +const T2Colon = 57374 +const TIdent = 57375 +const TNumber = 57376 +const TString = 57377 +const UNARY = 57378 + +var yyToknames = [...]string{ + "$end", + "error", + "$unk", + "TAnd", + "TBreak", + "TDo", + "TElse", + "TElseIf", + "TEnd", + "TFalse", + "TFor", + "TFunction", + "TIf", + "TIn", + "TLocal", + "TNil", + "TNot", + "TOr", + "TReturn", + "TRepeat", + "TThen", + "TTrue", + "TUntil", + "TWhile", + "TGoto", + "TEqeq", + "TNeq", + "TLte", + "TGte", + "T2Comma", + "T3Comma", + "T2Colon", + "TIdent", + "TNumber", + "TString", + "'{'", + "'('", + "'>'", + "'<'", + "'+'", + "'-'", + "'*'", + "'/'", + "'%'", + "UNARY", + "'^'", + "';'", + "'='", + "','", + "':'", + "'.'", + "'['", + "']'", + "'#'", + "')'", + "'}'", +} + +var yyStatenames = [...]string{} + +const yyEofCode = 1 +const yyErrCode = 2 +const yyInitialStackSize = 16 + +//line parser.go.y:525 + +func TokenName(c int) string { + if c >= TAnd && c-TAnd < len(yyToknames) { + if yyToknames[c-TAnd] != "" { + return yyToknames[c-TAnd] + } + } + return string([]byte{byte(c)}) +} + +//line yacctab:1 +var yyExca = [...]int8{ + -1, 1, + 1, -1, + -2, 0, + -1, 19, + 48, 33, + 49, 33, + -2, 70, + -1, 97, + 48, 34, + 49, 34, + -2, 70, +} + +const yyPrivate = 57344 + +const yyLast = 616 + +var yyAct = [...]uint8{ + 26, 92, 52, 25, 47, 88, 158, 58, 142, 118, + 141, 167, 54, 69, 56, 55, 35, 139, 41, 42, + 49, 160, 34, 67, 63, 137, 50, 64, 112, 113, + 69, 109, 51, 48, 46, 45, 147, 85, 86, 87, + 115, 110, 24, 95, 43, 44, 99, 96, 78, 136, + 50, 171, 143, 103, 69, 108, 51, 84, 79, 80, + 81, 82, 83, 89, 84, 111, 110, 41, 42, 49, + 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, + 129, 130, 131, 132, 133, 134, 81, 82, 83, 170, + 84, 153, 155, 33, 154, 144, 9, 138, 40, 23, + 153, 19, 62, 22, 114, 101, 146, 149, 148, 151, + 150, 71, 100, 152, 66, 50, 65, 61, 50, 157, + 156, 51, 57, 64, 51, 70, 116, 106, 192, 21, + 173, 174, 172, 76, 77, 75, 74, 78, 98, 159, + 189, 95, 161, 97, 162, 72, 73, 79, 80, 81, + 82, 83, 68, 84, 184, 183, 177, 169, 164, 104, + 140, 168, 117, 53, 1, 91, 135, 175, 32, 20, + 176, 8, 178, 60, 59, 180, 179, 3, 165, 28, + 4, 39, 2, 187, 186, 27, 37, 0, 188, 0, + 0, 29, 0, 191, 71, 0, 0, 0, 0, 0, + 31, 0, 93, 30, 41, 42, 22, 0, 70, 0, + 36, 0, 0, 0, 0, 0, 76, 77, 75, 74, + 78, 94, 0, 38, 71, 90, 0, 0, 72, 73, + 79, 80, 81, 82, 83, 0, 84, 0, 70, 0, + 0, 0, 0, 163, 0, 0, 76, 77, 75, 74, + 78, 0, 0, 0, 0, 0, 0, 0, 72, 73, + 79, 80, 81, 82, 83, 28, 84, 39, 0, 0, + 0, 27, 37, 145, 0, 0, 0, 29, 0, 0, + 0, 0, 0, 0, 0, 0, 31, 0, 23, 30, + 41, 42, 22, 28, 0, 39, 36, 0, 0, 27, + 37, 0, 0, 0, 0, 29, 0, 0, 0, 38, + 102, 0, 0, 0, 31, 0, 93, 30, 41, 42, + 22, 28, 0, 39, 36, 0, 0, 27, 37, 0, + 0, 0, 0, 29, 0, 94, 71, 38, 181, 0, + 0, 0, 31, 0, 23, 30, 41, 42, 22, 0, + 70, 0, 36, 0, 0, 0, 0, 0, 76, 77, + 75, 74, 78, 71, 0, 38, 0, 0, 0, 0, + 72, 73, 79, 80, 81, 82, 83, 70, 84, 0, + 0, 182, 0, 0, 0, 76, 77, 75, 74, 78, + 71, 0, 190, 0, 0, 0, 0, 72, 73, 79, + 80, 81, 82, 83, 70, 84, 0, 0, 166, 0, + 0, 0, 76, 77, 75, 74, 78, 71, 0, 0, + 0, 0, 0, 0, 72, 73, 79, 80, 81, 82, + 83, 70, 84, 0, 185, 0, 0, 0, 0, 76, + 77, 75, 74, 78, 71, 0, 0, 0, 0, 0, + 0, 72, 73, 79, 80, 81, 82, 83, 70, 84, + 0, 107, 0, 0, 0, 0, 76, 77, 75, 74, + 78, 71, 0, 105, 0, 0, 0, 0, 72, 73, + 79, 80, 81, 82, 83, 70, 84, 0, 0, 0, + 0, 0, 0, 76, 77, 75, 74, 78, 71, 0, + 0, 0, 0, 0, 0, 72, 73, 79, 80, 81, + 82, 83, 70, 84, 0, 0, 0, 0, 0, 0, + 76, 77, 75, 74, 78, 0, 0, 0, 0, 0, + 0, 0, 72, 73, 79, 80, 81, 82, 83, 0, + 84, 7, 10, 0, 0, 0, 0, 14, 15, 13, + 0, 16, 71, 0, 0, 6, 12, 0, 0, 0, + 11, 18, 0, 0, 0, 0, 0, 0, 17, 23, + 0, 0, 0, 22, 76, 77, 75, 74, 78, 0, + 0, 0, 0, 5, 0, 0, 72, 73, 79, 80, + 81, 82, 83, 0, 84, 76, 77, 75, 74, 78, + 0, 0, 0, 0, 0, 0, 0, 72, 73, 79, + 80, 81, 82, 83, 0, 84, +} + +var yyPact = [...]int16{ + -1000, -1000, 536, -5, -1000, -1000, 311, -1000, -4, -17, + -1000, 311, -1000, 311, 89, 84, 90, 83, 81, -1000, + -1000, -1000, 311, -1000, -1000, -36, 494, -1000, -1000, -1000, + -1000, -1000, -1000, -17, -1000, -1000, 311, 311, 311, 26, + -1000, -1000, 169, 311, 66, 311, 79, -1000, 72, 255, + -1000, -1000, 150, -1000, 467, 104, 440, 7, 17, 26, + -22, -1000, 71, -8, -1000, 94, -1000, 107, -46, 311, + 311, 311, 311, 311, 311, 311, 311, 311, 311, 311, + 311, 311, 311, 311, 311, 11, 11, 11, -1000, -6, + -1000, -39, -1000, 4, 311, 494, -36, -1000, -17, 220, + -1000, 32, -1000, -19, -1000, -1000, 311, -1000, 311, 311, + 67, -1000, 61, 59, 26, 311, -1000, -1000, -1000, 494, + 548, 569, 18, 18, 18, 18, 18, 18, 18, 44, + 44, 11, 11, 11, 11, -49, -1000, -1000, -28, -1000, + 283, -1000, -1000, 311, 190, -1000, -1000, -1000, 149, 494, + -1000, 359, 5, -1000, -1000, -1000, -1000, -36, -1000, 148, + 58, -1000, 494, 3, -1000, 123, 311, -1000, 147, -1000, + -1000, 311, -1000, -1000, 311, 332, 146, -1000, 494, 145, + 413, -1000, 311, -1000, -1000, -1000, 131, 386, -1000, -1000, + -1000, 119, -1000, +} + +var yyPgo = [...]uint8{ + 0, 163, 182, 2, 180, 178, 177, 174, 173, 171, + 98, 7, 3, 0, 22, 93, 129, 169, 4, 168, + 5, 166, 16, 165, 1, 160, +} + +var yyR1 = [...]int8{ + 0, 1, 1, 1, 2, 2, 2, 3, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 5, 5, 6, 6, 6, 7, + 7, 8, 8, 9, 9, 10, 10, 10, 11, 11, + 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 14, + 15, 15, 15, 15, 17, 16, 16, 18, 18, 18, + 18, 19, 20, 20, 21, 21, 21, 22, 22, 23, + 23, 23, 24, 24, 24, 25, 25, +} + +var yyR2 = [...]int8{ + 0, 1, 2, 3, 0, 2, 2, 1, 3, 1, + 3, 5, 4, 6, 8, 9, 11, 7, 3, 4, + 4, 2, 3, 2, 0, 5, 1, 2, 1, 1, + 3, 1, 3, 1, 3, 1, 4, 3, 1, 3, + 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 2, 2, 2, 1, + 1, 1, 1, 3, 3, 2, 4, 2, 3, 1, + 1, 2, 5, 4, 1, 1, 3, 2, 3, 1, + 3, 2, 3, 5, 1, 1, 1, +} + +var yyChk = [...]int16{ + -1000, -1, -2, -6, -4, 47, 19, 5, -9, -15, + 6, 24, 20, 13, 11, 12, 15, 32, 25, -10, + -17, -16, 37, 33, 47, -12, -13, 16, 10, 22, + 34, 31, -19, -15, -14, -22, 41, 17, 54, 12, + -10, 35, 36, 48, 49, 52, 51, -18, 50, 37, + -22, -14, -3, -1, -13, -3, -13, 33, -11, -7, + -8, 33, 12, -11, 33, 33, 33, -13, -16, 49, + 18, 4, 38, 39, 29, 28, 26, 27, 30, 40, + 41, 42, 43, 44, 46, -13, -13, -13, -20, 37, + 56, -23, -24, 33, 52, -13, -12, -10, -15, -13, + 33, 33, 55, -12, 9, 6, 23, 21, 48, 14, + 49, -20, 50, 51, 33, 48, 32, 55, 55, -13, + -13, -13, -13, -13, -13, -13, -13, -13, -13, -13, + -13, -13, -13, -13, -13, -21, 55, 31, -11, 56, + -25, 49, 47, 48, -13, 53, -18, 55, -3, -13, + -3, -13, -12, 33, 33, 33, -20, -12, 55, -3, + 49, -24, -13, 53, 9, -5, 49, 6, -3, 9, + 31, 48, 9, 7, 8, -13, -3, 9, -13, -3, + -13, 6, 49, 9, 9, 21, -3, -13, -3, 9, + 6, -3, 9, +} + +var yyDef = [...]int8{ + 4, -2, 1, 2, 5, 6, 26, 28, 0, 9, + 4, 0, 4, 0, 0, 0, 0, 0, 0, -2, + 71, 72, 0, 35, 3, 27, 40, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 0, 0, 0, 0, + 70, 69, 0, 0, 0, 0, 0, 75, 0, 0, + 79, 80, 0, 7, 0, 0, 0, 38, 0, 0, + 29, 31, 0, 21, 38, 0, 23, 0, 72, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 66, 67, 68, 81, 0, + 87, 0, 89, 35, 0, 94, 8, -2, 0, 0, + 37, 0, 77, 0, 10, 4, 0, 4, 0, 0, + 0, 18, 0, 0, 0, 0, 22, 73, 74, 41, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 62, 63, 64, 65, 0, 4, 84, 85, 88, + 91, 95, 96, 0, 0, 36, 76, 78, 0, 12, + 24, 0, 0, 39, 30, 32, 19, 20, 4, 0, + 0, 90, 92, 0, 11, 0, 0, 4, 0, 83, + 86, 0, 13, 4, 0, 0, 0, 82, 93, 0, + 0, 4, 0, 17, 14, 4, 0, 0, 25, 15, + 4, 0, 16, +} + +var yyTok1 = [...]int8{ + 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 54, 3, 44, 3, 3, + 37, 55, 42, 40, 49, 41, 51, 43, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 50, 47, + 39, 48, 38, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 52, 3, 53, 46, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 36, 3, 56, +} + +var yyTok2 = [...]int8{ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 45, +} + +var yyTok3 = [...]int8{ + 0, +} + +var yyErrorMessages = [...]struct { + state int + token int + msg string +}{} + +//line yaccpar:1 + +/* parser for yacc output */ + +var ( + yyDebug = 0 + yyErrorVerbose = false +) + +type yyLexer interface { + Lex(lval *yySymType) int + Error(s string) +} + +type yyParser interface { + Parse(yyLexer) int + Lookahead() int +} + +type yyParserImpl struct { + lval yySymType + stack [yyInitialStackSize]yySymType + char int +} + +func (p *yyParserImpl) Lookahead() int { + return p.char +} + +func yyNewParser() yyParser { + return &yyParserImpl{} +} + +const yyFlag = -1000 + +func yyTokname(c int) string { + if c >= 1 && c-1 < len(yyToknames) { + if yyToknames[c-1] != "" { + return yyToknames[c-1] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func yyStatname(s int) string { + if s >= 0 && s < len(yyStatenames) { + if yyStatenames[s] != "" { + return yyStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func yyErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !yyErrorVerbose { + return "syntax error" + } + + for _, e := range yyErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + yyTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := int(yyPact[state]) + for tok := TOKSTART; tok-1 < len(yyToknames); tok++ { + if n := base + tok; n >= 0 && n < yyLast && int(yyChk[int(yyAct[n])]) == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if yyDef[state] == -2 { + i := 0 + for yyExca[i] != -1 || int(yyExca[i+1]) != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; yyExca[i] >= 0; i += 2 { + tok := int(yyExca[i]) + if tok < TOKSTART || yyExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if yyExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += yyTokname(tok) + } + return res +} + +func yylex1(lex yyLexer, lval *yySymType) (char, token int) { + token = 0 + char = lex.Lex(lval) + if char <= 0 { + token = int(yyTok1[0]) + goto out + } + if char < len(yyTok1) { + token = int(yyTok1[char]) + goto out + } + if char >= yyPrivate { + if char < yyPrivate+len(yyTok2) { + token = int(yyTok2[char-yyPrivate]) + goto out + } + } + for i := 0; i < len(yyTok3); i += 2 { + token = int(yyTok3[i+0]) + if token == char { + token = int(yyTok3[i+1]) + goto out + } + } + +out: + if token == 0 { + token = int(yyTok2[1]) /* unknown char */ + } + if yyDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", yyTokname(token), uint(char)) + } + return char, token +} + +func yyParse(yylex yyLexer) int { + return yyNewParser().Parse(yylex) +} + +func (yyrcvr *yyParserImpl) Parse(yylex yyLexer) int { + var yyn int + var yyVAL yySymType + var yyDollar []yySymType + _ = yyDollar // silence set and not used + yyS := yyrcvr.stack[:] + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + yystate := 0 + yyrcvr.char = -1 + yytoken := -1 // yyrcvr.char translated into internal numbering + defer func() { + // Make sure we report no lookahead when not parsing. + yystate = -1 + yyrcvr.char = -1 + yytoken = -1 + }() + yyp := -1 + goto yystack + +ret0: + return 0 + +ret1: + return 1 + +yystack: + /* put a state and value onto the stack */ + if yyDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", yyTokname(yytoken), yyStatname(yystate)) + } + + yyp++ + if yyp >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyS[yyp] = yyVAL + yyS[yyp].yys = yystate + +yynewstate: + yyn = int(yyPact[yystate]) + if yyn <= yyFlag { + goto yydefault /* simple state */ + } + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + yyn += yytoken + if yyn < 0 || yyn >= yyLast { + goto yydefault + } + yyn = int(yyAct[yyn]) + if int(yyChk[yyn]) == yytoken { /* valid shift */ + yyrcvr.char = -1 + yytoken = -1 + yyVAL = yyrcvr.lval + yystate = yyn + if Errflag > 0 { + Errflag-- + } + goto yystack + } + +yydefault: + /* default state action */ + yyn = int(yyDef[yystate]) + if yyn == -2 { + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + + /* look through exception table */ + xi := 0 + for { + if yyExca[xi+0] == -1 && int(yyExca[xi+1]) == yystate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + yyn = int(yyExca[xi+0]) + if yyn < 0 || yyn == yytoken { + break + } + } + yyn = int(yyExca[xi+1]) + if yyn < 0 { + goto ret0 + } + } + if yyn == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + yylex.Error(yyErrorMessage(yystate, yytoken)) + Nerrs++ + if yyDebug >= 1 { + __yyfmt__.Printf("%s", yyStatname(yystate)) + __yyfmt__.Printf(" saw %s\n", yyTokname(yytoken)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for yyp >= 0 { + yyn = int(yyPact[yyS[yyp].yys]) + yyErrCode + if yyn >= 0 && yyn < yyLast { + yystate = int(yyAct[yyn]) /* simulate a shift of "error" */ + if int(yyChk[yystate]) == yyErrCode { + goto yystack + } + } + + /* the current p has no shift on "error", pop stack */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys) + } + yyp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", yyTokname(yytoken)) + } + if yytoken == yyEofCode { + goto ret1 + } + yyrcvr.char = -1 + yytoken = -1 + goto yynewstate /* try again in the same state */ + } + } + + /* reduction by production yyn */ + if yyDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate)) + } + + yynt := yyn + yypt := yyp + _ = yypt // guard against "declared and not used" + + yyp -= int(yyR2[yyn]) + // yyp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if yyp+1 >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyVAL = yyS[yyp+1] + + /* consult goto table to find next state */ + yyn = int(yyR1[yyn]) + yyg := int(yyPgo[yyn]) + yyj := yyg + yyS[yyp].yys + 1 + + if yyj >= yyLast { + yystate = int(yyAct[yyg]) + } else { + yystate = int(yyAct[yyj]) + if int(yyChk[yystate]) != -yyn { + yystate = int(yyAct[yyg]) + } + } + // dummy call; replaced with literal code + switch yynt { + + case 1: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:73 + { + yyVAL.stmts = yyDollar[1].stmts + if l, ok := yylex.(*Lexer); ok { + l.Stmts = yyVAL.stmts + } + } + case 2: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:79 + { + yyVAL.stmts = append(yyDollar[1].stmts, yyDollar[2].stmt) + if l, ok := yylex.(*Lexer); ok { + l.Stmts = yyVAL.stmts + } + } + case 3: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:85 + { + yyVAL.stmts = append(yyDollar[1].stmts, yyDollar[2].stmt) + if l, ok := yylex.(*Lexer); ok { + l.Stmts = yyVAL.stmts + } + } + case 4: + yyDollar = yyS[yypt-0 : yypt+1] +//line parser.go.y:93 + { + yyVAL.stmts = []ast.Stmt{} + } + case 5: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:96 + { + yyVAL.stmts = append(yyDollar[1].stmts, yyDollar[2].stmt) + } + case 6: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:99 + { + yyVAL.stmts = yyDollar[1].stmts + } + case 7: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:104 + { + yyVAL.stmts = yyDollar[1].stmts + } + case 8: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:109 + { + yyVAL.stmt = &ast.AssignStmt{Lhs: yyDollar[1].exprlist, Rhs: yyDollar[3].exprlist} + yyVAL.stmt.SetLine(yyDollar[1].exprlist[0].Line()) + } + case 9: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:114 + { + if _, ok := yyDollar[1].expr.(*ast.FuncCallExpr); !ok { + yylex.(*Lexer).Error("parse error") + } else { + yyVAL.stmt = &ast.FuncCallStmt{Expr: yyDollar[1].expr} + yyVAL.stmt.SetLine(yyDollar[1].expr.Line()) + } + } + case 10: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:122 + { + yyVAL.stmt = &ast.DoBlockStmt{Stmts: yyDollar[2].stmts} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[3].token.Pos.Line) + } + case 11: + yyDollar = yyS[yypt-5 : yypt+1] +//line parser.go.y:127 + { + yyVAL.stmt = &ast.WhileStmt{Condition: yyDollar[2].expr, Stmts: yyDollar[4].stmts} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[5].token.Pos.Line) + } + case 12: + yyDollar = yyS[yypt-4 : yypt+1] +//line parser.go.y:132 + { + yyVAL.stmt = &ast.RepeatStmt{Condition: yyDollar[4].expr, Stmts: yyDollar[2].stmts} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[4].expr.Line()) + } + case 13: + yyDollar = yyS[yypt-6 : yypt+1] +//line parser.go.y:137 + { + yyVAL.stmt = &ast.IfStmt{Condition: yyDollar[2].expr, Then: yyDollar[4].stmts} + cur := yyVAL.stmt + for _, elseif := range yyDollar[5].stmts { + cur.(*ast.IfStmt).Else = []ast.Stmt{elseif} + cur = elseif + } + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[6].token.Pos.Line) + } + case 14: + yyDollar = yyS[yypt-8 : yypt+1] +//line parser.go.y:147 + { + yyVAL.stmt = &ast.IfStmt{Condition: yyDollar[2].expr, Then: yyDollar[4].stmts} + cur := yyVAL.stmt + for _, elseif := range yyDollar[5].stmts { + cur.(*ast.IfStmt).Else = []ast.Stmt{elseif} + cur = elseif + } + cur.(*ast.IfStmt).Else = yyDollar[7].stmts + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[8].token.Pos.Line) + } + case 15: + yyDollar = yyS[yypt-9 : yypt+1] +//line parser.go.y:158 + { + yyVAL.stmt = &ast.NumberForStmt{Name: yyDollar[2].token.Str, Init: yyDollar[4].expr, Limit: yyDollar[6].expr, Stmts: yyDollar[8].stmts} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[9].token.Pos.Line) + } + case 16: + yyDollar = yyS[yypt-11 : yypt+1] +//line parser.go.y:163 + { + yyVAL.stmt = &ast.NumberForStmt{Name: yyDollar[2].token.Str, Init: yyDollar[4].expr, Limit: yyDollar[6].expr, Step: yyDollar[8].expr, Stmts: yyDollar[10].stmts} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[11].token.Pos.Line) + } + case 17: + yyDollar = yyS[yypt-7 : yypt+1] +//line parser.go.y:168 + { + yyVAL.stmt = &ast.GenericForStmt{Names: yyDollar[2].namelist, Exprs: yyDollar[4].exprlist, Stmts: yyDollar[6].stmts} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[7].token.Pos.Line) + } + case 18: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:173 + { + yyVAL.stmt = &ast.FuncDefStmt{Name: yyDollar[2].funcname, Func: yyDollar[3].funcexpr} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[3].funcexpr.LastLine()) + } + case 19: + yyDollar = yyS[yypt-4 : yypt+1] +//line parser.go.y:178 + { + yyVAL.stmt = &ast.LocalAssignStmt{Names: []string{yyDollar[3].token.Str}, Exprs: []ast.Expr{yyDollar[4].funcexpr}} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.stmt.SetLastLine(yyDollar[4].funcexpr.LastLine()) + } + case 20: + yyDollar = yyS[yypt-4 : yypt+1] +//line parser.go.y:183 + { + yyVAL.stmt = &ast.LocalAssignStmt{Names: yyDollar[2].namelist, Exprs: yyDollar[4].exprlist} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + } + case 21: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:187 + { + yyVAL.stmt = &ast.LocalAssignStmt{Names: yyDollar[2].namelist, Exprs: []ast.Expr{}} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + } + case 22: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:191 + { + yyVAL.stmt = &ast.LabelStmt{Name: yyDollar[2].token.Str} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + } + case 23: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:195 + { + yyVAL.stmt = &ast.GotoStmt{Label: yyDollar[2].token.Str} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + } + case 24: + yyDollar = yyS[yypt-0 : yypt+1] +//line parser.go.y:201 + { + yyVAL.stmts = []ast.Stmt{} + } + case 25: + yyDollar = yyS[yypt-5 : yypt+1] +//line parser.go.y:204 + { + yyVAL.stmts = append(yyDollar[1].stmts, &ast.IfStmt{Condition: yyDollar[3].expr, Then: yyDollar[5].stmts}) + yyVAL.stmts[len(yyVAL.stmts)-1].SetLine(yyDollar[2].token.Pos.Line) + } + case 26: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:210 + { + yyVAL.stmt = &ast.ReturnStmt{Exprs: nil} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + } + case 27: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:214 + { + yyVAL.stmt = &ast.ReturnStmt{Exprs: yyDollar[2].exprlist} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + } + case 28: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:218 + { + yyVAL.stmt = &ast.BreakStmt{} + yyVAL.stmt.SetLine(yyDollar[1].token.Pos.Line) + } + case 29: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:224 + { + yyVAL.funcname = yyDollar[1].funcname + } + case 30: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:227 + { + yyVAL.funcname = &ast.FuncName{Func: nil, Receiver: yyDollar[1].funcname.Func, Method: yyDollar[3].token.Str} + } + case 31: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:232 + { + yyVAL.funcname = &ast.FuncName{Func: &ast.IdentExpr{Value: yyDollar[1].token.Str}} + yyVAL.funcname.Func.SetLine(yyDollar[1].token.Pos.Line) + } + case 32: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:236 + { + key := &ast.StringExpr{Value: yyDollar[3].token.Str} + key.SetLine(yyDollar[3].token.Pos.Line) + fn := &ast.AttrGetExpr{Object: yyDollar[1].funcname.Func, Key: key} + fn.SetLine(yyDollar[3].token.Pos.Line) + yyVAL.funcname = &ast.FuncName{Func: fn} + } + case 33: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:245 + { + yyVAL.exprlist = []ast.Expr{yyDollar[1].expr} + } + case 34: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:248 + { + yyVAL.exprlist = append(yyDollar[1].exprlist, yyDollar[3].expr) + } + case 35: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:253 + { + yyVAL.expr = &ast.IdentExpr{Value: yyDollar[1].token.Str} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 36: + yyDollar = yyS[yypt-4 : yypt+1] +//line parser.go.y:257 + { + yyVAL.expr = &ast.AttrGetExpr{Object: yyDollar[1].expr, Key: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 37: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:261 + { + key := &ast.StringExpr{Value: yyDollar[3].token.Str} + key.SetLine(yyDollar[3].token.Pos.Line) + yyVAL.expr = &ast.AttrGetExpr{Object: yyDollar[1].expr, Key: key} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 38: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:269 + { + yyVAL.namelist = []string{yyDollar[1].token.Str} + } + case 39: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:272 + { + yyVAL.namelist = append(yyDollar[1].namelist, yyDollar[3].token.Str) + } + case 40: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:277 + { + yyVAL.exprlist = []ast.Expr{yyDollar[1].expr} + } + case 41: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:280 + { + yyVAL.exprlist = append(yyDollar[1].exprlist, yyDollar[3].expr) + } + case 42: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:285 + { + yyVAL.expr = &ast.NilExpr{} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 43: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:289 + { + yyVAL.expr = &ast.FalseExpr{} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 44: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:293 + { + yyVAL.expr = &ast.TrueExpr{} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 45: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:297 + { + yyVAL.expr = &ast.NumberExpr{Value: yyDollar[1].token.Str} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 46: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:301 + { + yyVAL.expr = &ast.Comma3Expr{} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 47: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:305 + { + yyVAL.expr = yyDollar[1].expr + } + case 48: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:308 + { + yyVAL.expr = yyDollar[1].expr + } + case 49: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:311 + { + yyVAL.expr = yyDollar[1].expr + } + case 50: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:314 + { + yyVAL.expr = yyDollar[1].expr + } + case 51: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:317 + { + yyVAL.expr = &ast.LogicalOpExpr{Lhs: yyDollar[1].expr, Operator: "or", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 52: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:321 + { + yyVAL.expr = &ast.LogicalOpExpr{Lhs: yyDollar[1].expr, Operator: "and", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 53: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:325 + { + yyVAL.expr = &ast.RelationalOpExpr{Lhs: yyDollar[1].expr, Operator: ">", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 54: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:329 + { + yyVAL.expr = &ast.RelationalOpExpr{Lhs: yyDollar[1].expr, Operator: "<", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 55: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:333 + { + yyVAL.expr = &ast.RelationalOpExpr{Lhs: yyDollar[1].expr, Operator: ">=", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 56: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:337 + { + yyVAL.expr = &ast.RelationalOpExpr{Lhs: yyDollar[1].expr, Operator: "<=", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 57: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:341 + { + yyVAL.expr = &ast.RelationalOpExpr{Lhs: yyDollar[1].expr, Operator: "==", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 58: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:345 + { + yyVAL.expr = &ast.RelationalOpExpr{Lhs: yyDollar[1].expr, Operator: "~=", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 59: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:349 + { + yyVAL.expr = &ast.StringConcatOpExpr{Lhs: yyDollar[1].expr, Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 60: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:353 + { + yyVAL.expr = &ast.ArithmeticOpExpr{Lhs: yyDollar[1].expr, Operator: "+", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 61: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:357 + { + yyVAL.expr = &ast.ArithmeticOpExpr{Lhs: yyDollar[1].expr, Operator: "-", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 62: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:361 + { + yyVAL.expr = &ast.ArithmeticOpExpr{Lhs: yyDollar[1].expr, Operator: "*", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 63: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:365 + { + yyVAL.expr = &ast.ArithmeticOpExpr{Lhs: yyDollar[1].expr, Operator: "/", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 64: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:369 + { + yyVAL.expr = &ast.ArithmeticOpExpr{Lhs: yyDollar[1].expr, Operator: "%", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 65: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:373 + { + yyVAL.expr = &ast.ArithmeticOpExpr{Lhs: yyDollar[1].expr, Operator: "^", Rhs: yyDollar[3].expr} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 66: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:377 + { + yyVAL.expr = &ast.UnaryMinusOpExpr{Expr: yyDollar[2].expr} + yyVAL.expr.SetLine(yyDollar[2].expr.Line()) + } + case 67: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:381 + { + yyVAL.expr = &ast.UnaryNotOpExpr{Expr: yyDollar[2].expr} + yyVAL.expr.SetLine(yyDollar[2].expr.Line()) + } + case 68: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:385 + { + yyVAL.expr = &ast.UnaryLenOpExpr{Expr: yyDollar[2].expr} + yyVAL.expr.SetLine(yyDollar[2].expr.Line()) + } + case 69: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:391 + { + yyVAL.expr = &ast.StringExpr{Value: yyDollar[1].token.Str} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 70: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:397 + { + yyVAL.expr = yyDollar[1].expr + } + case 71: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:400 + { + yyVAL.expr = yyDollar[1].expr + } + case 72: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:403 + { + yyVAL.expr = yyDollar[1].expr + } + case 73: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:406 + { + if ex, ok := yyDollar[2].expr.(*ast.Comma3Expr); ok { + ex.AdjustRet = true + } + yyVAL.expr = yyDollar[2].expr + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 74: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:415 + { + yyDollar[2].expr.(*ast.FuncCallExpr).AdjustRet = true + yyVAL.expr = yyDollar[2].expr + } + case 75: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:421 + { + yyVAL.expr = &ast.FuncCallExpr{Func: yyDollar[1].expr, Args: yyDollar[2].exprlist} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 76: + yyDollar = yyS[yypt-4 : yypt+1] +//line parser.go.y:425 + { + yyVAL.expr = &ast.FuncCallExpr{Method: yyDollar[3].token.Str, Receiver: yyDollar[1].expr, Args: yyDollar[4].exprlist} + yyVAL.expr.SetLine(yyDollar[1].expr.Line()) + } + case 77: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:431 + { + if yylex.(*Lexer).PNewLine { + yylex.(*Lexer).TokenError(yyDollar[1].token, "ambiguous syntax (function call x new statement)") + } + yyVAL.exprlist = []ast.Expr{} + } + case 78: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:437 + { + if yylex.(*Lexer).PNewLine { + yylex.(*Lexer).TokenError(yyDollar[1].token, "ambiguous syntax (function call x new statement)") + } + yyVAL.exprlist = yyDollar[2].exprlist + } + case 79: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:443 + { + yyVAL.exprlist = []ast.Expr{yyDollar[1].expr} + } + case 80: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:446 + { + yyVAL.exprlist = []ast.Expr{yyDollar[1].expr} + } + case 81: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:451 + { + yyVAL.expr = &ast.FunctionExpr{ParList: yyDollar[2].funcexpr.ParList, Stmts: yyDollar[2].funcexpr.Stmts} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.expr.SetLastLine(yyDollar[2].funcexpr.LastLine()) + } + case 82: + yyDollar = yyS[yypt-5 : yypt+1] +//line parser.go.y:458 + { + yyVAL.funcexpr = &ast.FunctionExpr{ParList: yyDollar[2].parlist, Stmts: yyDollar[4].stmts} + yyVAL.funcexpr.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.funcexpr.SetLastLine(yyDollar[5].token.Pos.Line) + } + case 83: + yyDollar = yyS[yypt-4 : yypt+1] +//line parser.go.y:463 + { + yyVAL.funcexpr = &ast.FunctionExpr{ParList: &ast.ParList{HasVargs: false, Names: []string{}}, Stmts: yyDollar[3].stmts} + yyVAL.funcexpr.SetLine(yyDollar[1].token.Pos.Line) + yyVAL.funcexpr.SetLastLine(yyDollar[4].token.Pos.Line) + } + case 84: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:470 + { + yyVAL.parlist = &ast.ParList{HasVargs: true, Names: []string{}} + } + case 85: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:473 + { + yyVAL.parlist = &ast.ParList{HasVargs: false, Names: []string{}} + yyVAL.parlist.Names = append(yyVAL.parlist.Names, yyDollar[1].namelist...) + } + case 86: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:477 + { + yyVAL.parlist = &ast.ParList{HasVargs: true, Names: []string{}} + yyVAL.parlist.Names = append(yyVAL.parlist.Names, yyDollar[1].namelist...) + } + case 87: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:484 + { + yyVAL.expr = &ast.TableExpr{Fields: []*ast.Field{}} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 88: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:488 + { + yyVAL.expr = &ast.TableExpr{Fields: yyDollar[2].fieldlist} + yyVAL.expr.SetLine(yyDollar[1].token.Pos.Line) + } + case 89: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:495 + { + yyVAL.fieldlist = []*ast.Field{yyDollar[1].field} + } + case 90: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:498 + { + yyVAL.fieldlist = append(yyDollar[1].fieldlist, yyDollar[3].field) + } + case 91: + yyDollar = yyS[yypt-2 : yypt+1] +//line parser.go.y:501 + { + yyVAL.fieldlist = yyDollar[1].fieldlist + } + case 92: + yyDollar = yyS[yypt-3 : yypt+1] +//line parser.go.y:506 + { + yyVAL.field = &ast.Field{Key: &ast.StringExpr{Value: yyDollar[1].token.Str}, Value: yyDollar[3].expr} + yyVAL.field.Key.SetLine(yyDollar[1].token.Pos.Line) + } + case 93: + yyDollar = yyS[yypt-5 : yypt+1] +//line parser.go.y:510 + { + yyVAL.field = &ast.Field{Key: yyDollar[2].expr, Value: yyDollar[5].expr} + } + case 94: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:513 + { + yyVAL.field = &ast.Field{Value: yyDollar[1].expr} + } + case 95: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:518 + { + yyVAL.fieldsep = "," + } + case 96: + yyDollar = yyS[yypt-1 : yypt+1] +//line parser.go.y:521 + { + yyVAL.fieldsep = ";" + } + } + goto yystack /* stack new state and value */ +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/parse/parser.go.y b/scraper-go/vendor/github.com/yuin/gopher-lua/parse/parser.go.y new file mode 100644 index 00000000..52bcd55f --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/parse/parser.go.y @@ -0,0 +1,535 @@ +%{ +package parse + +import ( + "github.com/yuin/gopher-lua/ast" +) +%} +%type chunk +%type chunk1 +%type block +%type stat +%type elseifs +%type laststat +%type funcname +%type funcname1 +%type varlist +%type var +%type namelist +%type exprlist +%type expr +%type string +%type prefixexp +%type functioncall +%type afunctioncall +%type args +%type function +%type funcbody +%type parlist +%type tableconstructor +%type fieldlist +%type field +%type fieldsep + +%union { + token ast.Token + + stmts []ast.Stmt + stmt ast.Stmt + + funcname *ast.FuncName + funcexpr *ast.FunctionExpr + + exprlist []ast.Expr + expr ast.Expr + + fieldlist []*ast.Field + field *ast.Field + fieldsep string + + namelist []string + parlist *ast.ParList +} + +/* Reserved words */ +%token TAnd TBreak TDo TElse TElseIf TEnd TFalse TFor TFunction TIf TIn TLocal TNil TNot TOr TReturn TRepeat TThen TTrue TUntil TWhile TGoto + +/* Literals */ +%token TEqeq TNeq TLte TGte T2Comma T3Comma T2Colon TIdent TNumber TString '{' '(' + +/* Operators */ +%left TOr +%left TAnd +%left '>' '<' TGte TLte TEqeq TNeq +%right T2Comma +%left '+' '-' +%left '*' '/' '%' +%right UNARY /* not # -(unary) */ +%right '^' + +%% + +chunk: + chunk1 { + $$ = $1 + if l, ok := yylex.(*Lexer); ok { + l.Stmts = $$ + } + } | + chunk1 laststat { + $$ = append($1, $2) + if l, ok := yylex.(*Lexer); ok { + l.Stmts = $$ + } + } | + chunk1 laststat ';' { + $$ = append($1, $2) + if l, ok := yylex.(*Lexer); ok { + l.Stmts = $$ + } + } + +chunk1: + { + $$ = []ast.Stmt{} + } | + chunk1 stat { + $$ = append($1, $2) + } | + chunk1 ';' { + $$ = $1 + } + +block: + chunk { + $$ = $1 + } + +stat: + varlist '=' exprlist { + $$ = &ast.AssignStmt{Lhs: $1, Rhs: $3} + $$.SetLine($1[0].Line()) + } | + /* 'stat = functioncal' causes a reduce/reduce conflict */ + prefixexp { + if _, ok := $1.(*ast.FuncCallExpr); !ok { + yylex.(*Lexer).Error("parse error") + } else { + $$ = &ast.FuncCallStmt{Expr: $1} + $$.SetLine($1.Line()) + } + } | + TDo block TEnd { + $$ = &ast.DoBlockStmt{Stmts: $2} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($3.Pos.Line) + } | + TWhile expr TDo block TEnd { + $$ = &ast.WhileStmt{Condition: $2, Stmts: $4} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($5.Pos.Line) + } | + TRepeat block TUntil expr { + $$ = &ast.RepeatStmt{Condition: $4, Stmts: $2} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($4.Line()) + } | + TIf expr TThen block elseifs TEnd { + $$ = &ast.IfStmt{Condition: $2, Then: $4} + cur := $$ + for _, elseif := range $5 { + cur.(*ast.IfStmt).Else = []ast.Stmt{elseif} + cur = elseif + } + $$.SetLine($1.Pos.Line) + $$.SetLastLine($6.Pos.Line) + } | + TIf expr TThen block elseifs TElse block TEnd { + $$ = &ast.IfStmt{Condition: $2, Then: $4} + cur := $$ + for _, elseif := range $5 { + cur.(*ast.IfStmt).Else = []ast.Stmt{elseif} + cur = elseif + } + cur.(*ast.IfStmt).Else = $7 + $$.SetLine($1.Pos.Line) + $$.SetLastLine($8.Pos.Line) + } | + TFor TIdent '=' expr ',' expr TDo block TEnd { + $$ = &ast.NumberForStmt{Name: $2.Str, Init: $4, Limit: $6, Stmts: $8} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($9.Pos.Line) + } | + TFor TIdent '=' expr ',' expr ',' expr TDo block TEnd { + $$ = &ast.NumberForStmt{Name: $2.Str, Init: $4, Limit: $6, Step:$8, Stmts: $10} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($11.Pos.Line) + } | + TFor namelist TIn exprlist TDo block TEnd { + $$ = &ast.GenericForStmt{Names:$2, Exprs:$4, Stmts: $6} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($7.Pos.Line) + } | + TFunction funcname funcbody { + $$ = &ast.FuncDefStmt{Name: $2, Func: $3} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($3.LastLine()) + } | + TLocal TFunction TIdent funcbody { + $$ = &ast.LocalAssignStmt{Names:[]string{$3.Str}, Exprs: []ast.Expr{$4}} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($4.LastLine()) + } | + TLocal namelist '=' exprlist { + $$ = &ast.LocalAssignStmt{Names: $2, Exprs:$4} + $$.SetLine($1.Pos.Line) + } | + TLocal namelist { + $$ = &ast.LocalAssignStmt{Names: $2, Exprs:[]ast.Expr{}} + $$.SetLine($1.Pos.Line) + } | + T2Colon TIdent T2Colon { + $$ = &ast.LabelStmt{Name: $2.Str} + $$.SetLine($1.Pos.Line) + } | + TGoto TIdent { + $$ = &ast.GotoStmt{Label: $2.Str} + $$.SetLine($1.Pos.Line) + } + +elseifs: + { + $$ = []ast.Stmt{} + } | + elseifs TElseIf expr TThen block { + $$ = append($1, &ast.IfStmt{Condition: $3, Then: $5}) + $$[len($$)-1].SetLine($2.Pos.Line) + } + +laststat: + TReturn { + $$ = &ast.ReturnStmt{Exprs:nil} + $$.SetLine($1.Pos.Line) + } | + TReturn exprlist { + $$ = &ast.ReturnStmt{Exprs:$2} + $$.SetLine($1.Pos.Line) + } | + TBreak { + $$ = &ast.BreakStmt{} + $$.SetLine($1.Pos.Line) + } + +funcname: + funcname1 { + $$ = $1 + } | + funcname1 ':' TIdent { + $$ = &ast.FuncName{Func:nil, Receiver:$1.Func, Method: $3.Str} + } + +funcname1: + TIdent { + $$ = &ast.FuncName{Func: &ast.IdentExpr{Value:$1.Str}} + $$.Func.SetLine($1.Pos.Line) + } | + funcname1 '.' TIdent { + key:= &ast.StringExpr{Value:$3.Str} + key.SetLine($3.Pos.Line) + fn := &ast.AttrGetExpr{Object: $1.Func, Key: key} + fn.SetLine($3.Pos.Line) + $$ = &ast.FuncName{Func: fn} + } + +varlist: + var { + $$ = []ast.Expr{$1} + } | + varlist ',' var { + $$ = append($1, $3) + } + +var: + TIdent { + $$ = &ast.IdentExpr{Value:$1.Str} + $$.SetLine($1.Pos.Line) + } | + prefixexp '[' expr ']' { + $$ = &ast.AttrGetExpr{Object: $1, Key: $3} + $$.SetLine($1.Line()) + } | + prefixexp '.' TIdent { + key := &ast.StringExpr{Value:$3.Str} + key.SetLine($3.Pos.Line) + $$ = &ast.AttrGetExpr{Object: $1, Key: key} + $$.SetLine($1.Line()) + } + +namelist: + TIdent { + $$ = []string{$1.Str} + } | + namelist ',' TIdent { + $$ = append($1, $3.Str) + } + +exprlist: + expr { + $$ = []ast.Expr{$1} + } | + exprlist ',' expr { + $$ = append($1, $3) + } + +expr: + TNil { + $$ = &ast.NilExpr{} + $$.SetLine($1.Pos.Line) + } | + TFalse { + $$ = &ast.FalseExpr{} + $$.SetLine($1.Pos.Line) + } | + TTrue { + $$ = &ast.TrueExpr{} + $$.SetLine($1.Pos.Line) + } | + TNumber { + $$ = &ast.NumberExpr{Value: $1.Str} + $$.SetLine($1.Pos.Line) + } | + T3Comma { + $$ = &ast.Comma3Expr{} + $$.SetLine($1.Pos.Line) + } | + function { + $$ = $1 + } | + prefixexp { + $$ = $1 + } | + string { + $$ = $1 + } | + tableconstructor { + $$ = $1 + } | + expr TOr expr { + $$ = &ast.LogicalOpExpr{Lhs: $1, Operator: "or", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr TAnd expr { + $$ = &ast.LogicalOpExpr{Lhs: $1, Operator: "and", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr '>' expr { + $$ = &ast.RelationalOpExpr{Lhs: $1, Operator: ">", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr '<' expr { + $$ = &ast.RelationalOpExpr{Lhs: $1, Operator: "<", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr TGte expr { + $$ = &ast.RelationalOpExpr{Lhs: $1, Operator: ">=", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr TLte expr { + $$ = &ast.RelationalOpExpr{Lhs: $1, Operator: "<=", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr TEqeq expr { + $$ = &ast.RelationalOpExpr{Lhs: $1, Operator: "==", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr TNeq expr { + $$ = &ast.RelationalOpExpr{Lhs: $1, Operator: "~=", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr T2Comma expr { + $$ = &ast.StringConcatOpExpr{Lhs: $1, Rhs: $3} + $$.SetLine($1.Line()) + } | + expr '+' expr { + $$ = &ast.ArithmeticOpExpr{Lhs: $1, Operator: "+", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr '-' expr { + $$ = &ast.ArithmeticOpExpr{Lhs: $1, Operator: "-", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr '*' expr { + $$ = &ast.ArithmeticOpExpr{Lhs: $1, Operator: "*", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr '/' expr { + $$ = &ast.ArithmeticOpExpr{Lhs: $1, Operator: "/", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr '%' expr { + $$ = &ast.ArithmeticOpExpr{Lhs: $1, Operator: "%", Rhs: $3} + $$.SetLine($1.Line()) + } | + expr '^' expr { + $$ = &ast.ArithmeticOpExpr{Lhs: $1, Operator: "^", Rhs: $3} + $$.SetLine($1.Line()) + } | + '-' expr %prec UNARY { + $$ = &ast.UnaryMinusOpExpr{Expr: $2} + $$.SetLine($2.Line()) + } | + TNot expr %prec UNARY { + $$ = &ast.UnaryNotOpExpr{Expr: $2} + $$.SetLine($2.Line()) + } | + '#' expr %prec UNARY { + $$ = &ast.UnaryLenOpExpr{Expr: $2} + $$.SetLine($2.Line()) + } + +string: + TString { + $$ = &ast.StringExpr{Value: $1.Str} + $$.SetLine($1.Pos.Line) + } + +prefixexp: + var { + $$ = $1 + } | + afunctioncall { + $$ = $1 + } | + functioncall { + $$ = $1 + } | + '(' expr ')' { + if ex, ok := $2.(*ast.Comma3Expr); ok { + ex.AdjustRet = true + } + $$ = $2 + $$.SetLine($1.Pos.Line) + } + +afunctioncall: + '(' functioncall ')' { + $2.(*ast.FuncCallExpr).AdjustRet = true + $$ = $2 + } + +functioncall: + prefixexp args { + $$ = &ast.FuncCallExpr{Func: $1, Args: $2} + $$.SetLine($1.Line()) + } | + prefixexp ':' TIdent args { + $$ = &ast.FuncCallExpr{Method: $3.Str, Receiver: $1, Args: $4} + $$.SetLine($1.Line()) + } + +args: + '(' ')' { + if yylex.(*Lexer).PNewLine { + yylex.(*Lexer).TokenError($1, "ambiguous syntax (function call x new statement)") + } + $$ = []ast.Expr{} + } | + '(' exprlist ')' { + if yylex.(*Lexer).PNewLine { + yylex.(*Lexer).TokenError($1, "ambiguous syntax (function call x new statement)") + } + $$ = $2 + } | + tableconstructor { + $$ = []ast.Expr{$1} + } | + string { + $$ = []ast.Expr{$1} + } + +function: + TFunction funcbody { + $$ = &ast.FunctionExpr{ParList:$2.ParList, Stmts: $2.Stmts} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($2.LastLine()) + } + +funcbody: + '(' parlist ')' block TEnd { + $$ = &ast.FunctionExpr{ParList: $2, Stmts: $4} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($5.Pos.Line) + } | + '(' ')' block TEnd { + $$ = &ast.FunctionExpr{ParList: &ast.ParList{HasVargs: false, Names: []string{}}, Stmts: $3} + $$.SetLine($1.Pos.Line) + $$.SetLastLine($4.Pos.Line) + } + +parlist: + T3Comma { + $$ = &ast.ParList{HasVargs: true, Names: []string{}} + } | + namelist { + $$ = &ast.ParList{HasVargs: false, Names: []string{}} + $$.Names = append($$.Names, $1...) + } | + namelist ',' T3Comma { + $$ = &ast.ParList{HasVargs: true, Names: []string{}} + $$.Names = append($$.Names, $1...) + } + + +tableconstructor: + '{' '}' { + $$ = &ast.TableExpr{Fields: []*ast.Field{}} + $$.SetLine($1.Pos.Line) + } | + '{' fieldlist '}' { + $$ = &ast.TableExpr{Fields: $2} + $$.SetLine($1.Pos.Line) + } + + +fieldlist: + field { + $$ = []*ast.Field{$1} + } | + fieldlist fieldsep field { + $$ = append($1, $3) + } | + fieldlist fieldsep { + $$ = $1 + } + +field: + TIdent '=' expr { + $$ = &ast.Field{Key: &ast.StringExpr{Value:$1.Str}, Value: $3} + $$.Key.SetLine($1.Pos.Line) + } | + '[' expr ']' '=' expr { + $$ = &ast.Field{Key: $2, Value: $5} + } | + expr { + $$ = &ast.Field{Value: $1} + } + +fieldsep: + ',' { + $$ = "," + } | + ';' { + $$ = ";" + } + +%% + +func TokenName(c int) string { + if c >= TAnd && c-TAnd < len(yyToknames) { + if yyToknames[c-TAnd] != "" { + return yyToknames[c-TAnd] + } + } + return string([]byte{byte(c)}) +} + diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/pm/pm.go b/scraper-go/vendor/github.com/yuin/gopher-lua/pm/pm.go new file mode 100644 index 00000000..e5c651f9 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/pm/pm.go @@ -0,0 +1,638 @@ +// Lua pattern match functions for Go +package pm + +import ( + "fmt" +) + +const EOS = -1 +const _UNKNOWN = -2 + +/* Error {{{ */ + +type Error struct { + Pos int + Message string +} + +func newError(pos int, message string, args ...interface{}) *Error { + if len(args) == 0 { + return &Error{pos, message} + } + return &Error{pos, fmt.Sprintf(message, args...)} +} + +func (e *Error) Error() string { + switch e.Pos { + case EOS: + return fmt.Sprintf("%s at EOS", e.Message) + case _UNKNOWN: + return fmt.Sprintf("%s", e.Message) + default: + return fmt.Sprintf("%s at %d", e.Message, e.Pos) + } +} + +/* }}} */ + +/* MatchData {{{ */ + +type MatchData struct { + // captured positions + // layout + // xxxx xxxx xxxx xxx0 : caputured positions + // xxxx xxxx xxxx xxx1 : position captured positions + captures []uint32 +} + +func newMatchState() *MatchData { return &MatchData{[]uint32{}} } + +func (st *MatchData) addPosCapture(s, pos int) { + for s+1 >= len(st.captures) { + st.captures = append(st.captures, 0) + } + st.captures[s] = (uint32(pos) << 1) | 1 + st.captures[s+1] = (uint32(pos) << 1) | 1 +} + +func (st *MatchData) setCapture(s, pos int) uint32 { + for s >= len(st.captures) { + st.captures = append(st.captures, 0) + } + v := st.captures[s] + st.captures[s] = (uint32(pos) << 1) + return v +} + +func (st *MatchData) restoreCapture(s int, pos uint32) { st.captures[s] = pos } + +func (st *MatchData) CaptureLength() int { return len(st.captures) } + +func (st *MatchData) IsPosCapture(idx int) bool { return (st.captures[idx] & 1) == 1 } + +func (st *MatchData) Capture(idx int) int { return int(st.captures[idx] >> 1) } + +/* }}} */ + +/* scanner {{{ */ + +type scannerState struct { + Pos int + started bool +} + +type scanner struct { + src []byte + State scannerState + saved scannerState +} + +func newScanner(src []byte) *scanner { + return &scanner{ + src: src, + State: scannerState{ + Pos: 0, + started: false, + }, + saved: scannerState{}, + } +} + +func (sc *scanner) Length() int { return len(sc.src) } + +func (sc *scanner) Next() int { + if !sc.State.started { + sc.State.started = true + if len(sc.src) == 0 { + sc.State.Pos = EOS + } + } else { + sc.State.Pos = sc.NextPos() + } + if sc.State.Pos == EOS { + return EOS + } + return int(sc.src[sc.State.Pos]) +} + +func (sc *scanner) CurrentPos() int { + return sc.State.Pos +} + +func (sc *scanner) NextPos() int { + if sc.State.Pos == EOS || sc.State.Pos >= len(sc.src)-1 { + return EOS + } + if !sc.State.started { + return 0 + } else { + return sc.State.Pos + 1 + } +} + +func (sc *scanner) Peek() int { + cureof := sc.State.Pos == EOS + ch := sc.Next() + if !cureof { + if sc.State.Pos == EOS { + sc.State.Pos = len(sc.src) - 1 + } else { + sc.State.Pos-- + if sc.State.Pos < 0 { + sc.State.Pos = 0 + sc.State.started = false + } + } + } + return ch +} + +func (sc *scanner) Save() { sc.saved = sc.State } + +func (sc *scanner) Restore() { sc.State = sc.saved } + +/* }}} */ + +/* bytecode {{{ */ + +type opCode int + +const ( + opChar opCode = iota + opMatch + opTailMatch + opJmp + opSplit + opSave + opPSave + opBrace + opNumber +) + +type inst struct { + OpCode opCode + Class class + Operand1 int + Operand2 int +} + +/* }}} */ + +/* classes {{{ */ + +type class interface { + Matches(ch int) bool +} + +type dotClass struct{} + +func (pn *dotClass) Matches(ch int) bool { return true } + +type charClass struct { + Ch int +} + +func (pn *charClass) Matches(ch int) bool { return pn.Ch == ch } + +type singleClass struct { + Class int +} + +func (pn *singleClass) Matches(ch int) bool { + ret := false + switch pn.Class { + case 'a', 'A': + ret = 'A' <= ch && ch <= 'Z' || 'a' <= ch && ch <= 'z' + case 'c', 'C': + ret = (0x00 <= ch && ch <= 0x1F) || ch == 0x7F + case 'd', 'D': + ret = '0' <= ch && ch <= '9' + case 'l', 'L': + ret = 'a' <= ch && ch <= 'z' + case 'p', 'P': + ret = (0x21 <= ch && ch <= 0x2f) || (0x3a <= ch && ch <= 0x40) || (0x5b <= ch && ch <= 0x60) || (0x7b <= ch && ch <= 0x7e) + case 's', 'S': + switch ch { + case ' ', '\f', '\n', '\r', '\t', '\v': + ret = true + } + case 'u', 'U': + ret = 'A' <= ch && ch <= 'Z' + case 'w', 'W': + ret = '0' <= ch && ch <= '9' || 'A' <= ch && ch <= 'Z' || 'a' <= ch && ch <= 'z' + case 'x', 'X': + ret = '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F' + case 'z', 'Z': + ret = ch == 0 + default: + return ch == pn.Class + } + if 'A' <= pn.Class && pn.Class <= 'Z' { + return !ret + } + return ret +} + +type setClass struct { + IsNot bool + Classes []class +} + +func (pn *setClass) Matches(ch int) bool { + for _, class := range pn.Classes { + if class.Matches(ch) { + return !pn.IsNot + } + } + return pn.IsNot +} + +type rangeClass struct { + Begin class + End class +} + +func (pn *rangeClass) Matches(ch int) bool { + switch begin := pn.Begin.(type) { + case *charClass: + end, ok := pn.End.(*charClass) + if !ok { + return false + } + return begin.Ch <= ch && ch <= end.Ch + } + return false +} + +// }}} + +// patterns {{{ + +type pattern interface{} + +type singlePattern struct { + Class class +} + +type seqPattern struct { + MustHead bool + MustTail bool + Patterns []pattern +} + +type repeatPattern struct { + Type int + Class class +} + +type posCapPattern struct{} + +type capPattern struct { + Pattern pattern +} + +type numberPattern struct { + N int +} + +type bracePattern struct { + Begin int + End int +} + +// }}} + +/* parse {{{ */ + +func parseClass(sc *scanner, allowset bool) class { + ch := sc.Next() + switch ch { + case '%': + return &singleClass{sc.Next()} + case '.': + if allowset { + return &dotClass{} + } + return &charClass{ch} + case '[': + if allowset { + return parseClassSet(sc) + } + return &charClass{ch} + //case '^' '$', '(', ')', ']', '*', '+', '-', '?': + // panic(newError(sc.CurrentPos(), "invalid %c", ch)) + case EOS: + panic(newError(sc.CurrentPos(), "unexpected EOS")) + default: + return &charClass{ch} + } +} + +func parseClassSet(sc *scanner) class { + set := &setClass{false, []class{}} + if sc.Peek() == '^' { + set.IsNot = true + sc.Next() + } + isrange := false + for { + ch := sc.Peek() + switch ch { + // case '[': + // panic(newError(sc.CurrentPos(), "'[' can not be nested")) + case EOS: + panic(newError(sc.CurrentPos(), "unexpected EOS")) + case ']': + if len(set.Classes) > 0 { + sc.Next() + goto exit + } + fallthrough + case '-': + if len(set.Classes) > 0 { + sc.Next() + isrange = true + continue + } + fallthrough + default: + set.Classes = append(set.Classes, parseClass(sc, false)) + } + if isrange { + begin := set.Classes[len(set.Classes)-2] + end := set.Classes[len(set.Classes)-1] + set.Classes = set.Classes[0 : len(set.Classes)-2] + set.Classes = append(set.Classes, &rangeClass{begin, end}) + isrange = false + } + } +exit: + if isrange { + set.Classes = append(set.Classes, &charClass{'-'}) + } + + return set +} + +func parsePattern(sc *scanner, toplevel bool) *seqPattern { + pat := &seqPattern{} + if toplevel { + if sc.Peek() == '^' { + sc.Next() + pat.MustHead = true + } + } + for { + ch := sc.Peek() + switch ch { + case '%': + sc.Save() + sc.Next() + switch sc.Peek() { + case '0': + panic(newError(sc.CurrentPos(), "invalid capture index")) + case '1', '2', '3', '4', '5', '6', '7', '8', '9': + pat.Patterns = append(pat.Patterns, &numberPattern{sc.Next() - 48}) + case 'b': + sc.Next() + pat.Patterns = append(pat.Patterns, &bracePattern{sc.Next(), sc.Next()}) + default: + sc.Restore() + pat.Patterns = append(pat.Patterns, &singlePattern{parseClass(sc, true)}) + } + case '.', '[', ']': + pat.Patterns = append(pat.Patterns, &singlePattern{parseClass(sc, true)}) + //case ']': + // panic(newError(sc.CurrentPos(), "invalid ']'")) + case ')': + if toplevel { + panic(newError(sc.CurrentPos(), "invalid ')'")) + } + return pat + case '(': + sc.Next() + if sc.Peek() == ')' { + sc.Next() + pat.Patterns = append(pat.Patterns, &posCapPattern{}) + } else { + ret := &capPattern{parsePattern(sc, false)} + if sc.Peek() != ')' { + panic(newError(sc.CurrentPos(), "unfinished capture")) + } + sc.Next() + pat.Patterns = append(pat.Patterns, ret) + } + case '*', '+', '-', '?': + sc.Next() + if len(pat.Patterns) > 0 { + spat, ok := pat.Patterns[len(pat.Patterns)-1].(*singlePattern) + if ok { + pat.Patterns = pat.Patterns[0 : len(pat.Patterns)-1] + pat.Patterns = append(pat.Patterns, &repeatPattern{ch, spat.Class}) + continue + } + } + pat.Patterns = append(pat.Patterns, &singlePattern{&charClass{ch}}) + case '$': + if toplevel && (sc.NextPos() == sc.Length()-1 || sc.NextPos() == EOS) { + pat.MustTail = true + } else { + pat.Patterns = append(pat.Patterns, &singlePattern{&charClass{ch}}) + } + sc.Next() + case EOS: + sc.Next() + goto exit + default: + sc.Next() + pat.Patterns = append(pat.Patterns, &singlePattern{&charClass{ch}}) + } + } +exit: + return pat +} + +type iptr struct { + insts []inst + capture int +} + +func compilePattern(p pattern, ps ...*iptr) []inst { + var ptr *iptr + toplevel := false + if len(ps) == 0 { + toplevel = true + ptr = &iptr{[]inst{inst{opSave, nil, 0, -1}}, 2} + } else { + ptr = ps[0] + } + switch pat := p.(type) { + case *singlePattern: + ptr.insts = append(ptr.insts, inst{opChar, pat.Class, -1, -1}) + case *seqPattern: + for _, cp := range pat.Patterns { + compilePattern(cp, ptr) + } + case *repeatPattern: + idx := len(ptr.insts) + switch pat.Type { + case '*': + ptr.insts = append(ptr.insts, + inst{opSplit, nil, idx + 1, idx + 3}, + inst{opChar, pat.Class, -1, -1}, + inst{opJmp, nil, idx, -1}) + case '+': + ptr.insts = append(ptr.insts, + inst{opChar, pat.Class, -1, -1}, + inst{opSplit, nil, idx, idx + 2}) + case '-': + ptr.insts = append(ptr.insts, + inst{opSplit, nil, idx + 3, idx + 1}, + inst{opChar, pat.Class, -1, -1}, + inst{opJmp, nil, idx, -1}) + case '?': + ptr.insts = append(ptr.insts, + inst{opSplit, nil, idx + 1, idx + 2}, + inst{opChar, pat.Class, -1, -1}) + } + case *posCapPattern: + ptr.insts = append(ptr.insts, inst{opPSave, nil, ptr.capture, -1}) + ptr.capture += 2 + case *capPattern: + c0, c1 := ptr.capture, ptr.capture+1 + ptr.capture += 2 + ptr.insts = append(ptr.insts, inst{opSave, nil, c0, -1}) + compilePattern(pat.Pattern, ptr) + ptr.insts = append(ptr.insts, inst{opSave, nil, c1, -1}) + case *bracePattern: + ptr.insts = append(ptr.insts, inst{opBrace, nil, pat.Begin, pat.End}) + case *numberPattern: + ptr.insts = append(ptr.insts, inst{opNumber, nil, pat.N, -1}) + } + if toplevel { + if p.(*seqPattern).MustTail { + ptr.insts = append(ptr.insts, inst{opSave, nil, 1, -1}, inst{opTailMatch, nil, -1, -1}) + } + ptr.insts = append(ptr.insts, inst{opSave, nil, 1, -1}, inst{opMatch, nil, -1, -1}) + } + return ptr.insts +} + +/* }}} parse */ + +/* VM {{{ */ + +// Simple recursive virtual machine based on the +// "Regular Expression Matching: the Virtual Machine Approach" (https://swtch.com/~rsc/regexp/regexp2.html) +func recursiveVM(src []byte, insts []inst, pc, sp int, ms ...*MatchData) (bool, int, *MatchData) { + var m *MatchData + if len(ms) == 0 { + m = newMatchState() + } else { + m = ms[0] + } +redo: + inst := insts[pc] + switch inst.OpCode { + case opChar: + if sp >= len(src) || !inst.Class.Matches(int(src[sp])) { + return false, sp, m + } + pc++ + sp++ + goto redo + case opMatch: + return true, sp, m + case opTailMatch: + return sp >= len(src), sp, m + case opJmp: + pc = inst.Operand1 + goto redo + case opSplit: + if ok, nsp, _ := recursiveVM(src, insts, inst.Operand1, sp, m); ok { + return true, nsp, m + } + pc = inst.Operand2 + goto redo + case opSave: + s := m.setCapture(inst.Operand1, sp) + if ok, nsp, _ := recursiveVM(src, insts, pc+1, sp, m); ok { + return true, nsp, m + } + m.restoreCapture(inst.Operand1, s) + return false, sp, m + case opPSave: + m.addPosCapture(inst.Operand1, sp+1) + pc++ + goto redo + case opBrace: + if sp >= len(src) || int(src[sp]) != inst.Operand1 { + return false, sp, m + } + count := 1 + for sp = sp + 1; sp < len(src); sp++ { + if int(src[sp]) == inst.Operand2 { + count-- + } + if count == 0 { + pc++ + sp++ + goto redo + } + if int(src[sp]) == inst.Operand1 { + count++ + } + } + return false, sp, m + case opNumber: + idx := inst.Operand1 * 2 + if idx >= m.CaptureLength()-1 { + panic(newError(_UNKNOWN, "invalid capture index")) + } + capture := src[m.Capture(idx):m.Capture(idx+1)] + for i := 0; i < len(capture); i++ { + if i+sp >= len(src) || capture[i] != src[i+sp] { + return false, sp, m + } + } + pc++ + sp += len(capture) + goto redo + } + panic("should not reach here") +} + +/* }}} */ + +/* API {{{ */ + +func Find(p string, src []byte, offset, limit int) (matches []*MatchData, err error) { + defer func() { + if v := recover(); v != nil { + if perr, ok := v.(*Error); ok { + err = perr + } else { + panic(v) + } + } + }() + pat := parsePattern(newScanner([]byte(p)), true) + insts := compilePattern(pat) + matches = []*MatchData{} + for sp := offset; sp <= len(src); { + ok, nsp, ms := recursiveVM(src, insts, 0, sp) + sp++ + if ok { + if sp < nsp { + sp = nsp + } + matches = append(matches, ms) + } + if len(matches) == limit || pat.MustHead { + break + } + } + return +} + +/* }}} */ diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/state.go b/scraper-go/vendor/github.com/yuin/gopher-lua/state.go new file mode 100644 index 00000000..292f93b4 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/state.go @@ -0,0 +1,2306 @@ +package lua + +//////////////////////////////////////////////////////// +// This file was generated by go-inline. DO NOT EDIT. // +//////////////////////////////////////////////////////// + +import ( + "context" + "fmt" + "io" + "math" + "os" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/yuin/gopher-lua/parse" +) + +const MultRet = -1 +const RegistryIndex = -10000 +const EnvironIndex = -10001 +const GlobalsIndex = -10002 + +/* ApiError {{{ */ + +type ApiError struct { + Type ApiErrorType + Object LValue + StackTrace string + // Underlying error. This attribute is set only if the Type is ApiErrorFile or ApiErrorSyntax + Cause error +} + +func newApiError(code ApiErrorType, object LValue) *ApiError { + return &ApiError{code, object, "", nil} +} + +func newApiErrorS(code ApiErrorType, message string) *ApiError { + return newApiError(code, LString(message)) +} + +func newApiErrorE(code ApiErrorType, err error) *ApiError { + return &ApiError{code, LString(err.Error()), "", err} +} + +func (e *ApiError) Error() string { + if len(e.StackTrace) > 0 { + return fmt.Sprintf("%s\n%s", e.Object.String(), e.StackTrace) + } + return e.Object.String() +} + +type ApiErrorType int + +const ( + ApiErrorSyntax ApiErrorType = iota + ApiErrorFile + ApiErrorRun + ApiErrorError + ApiErrorPanic +) + +/* }}} */ + +/* ResumeState {{{ */ + +type ResumeState int + +const ( + ResumeOK ResumeState = iota + ResumeYield + ResumeError +) + +/* }}} */ + +/* P {{{ */ + +type P struct { + Fn LValue + NRet int + Protect bool + Handler *LFunction +} + +/* }}} */ + +/* Options {{{ */ + +// Options is a configuration that is used to create a new LState. +type Options struct { + // Call stack size. This defaults to `lua.CallStackSize`. + CallStackSize int + // Data stack size. This defaults to `lua.RegistrySize`. + RegistrySize int + // Allow the registry to grow from the registry size specified up to a value of RegistryMaxSize. A value of 0 + // indicates no growth is permitted. The registry will not shrink again after any growth. + RegistryMaxSize int + // If growth is enabled, step up by an additional `RegistryGrowStep` each time to avoid having to resize too often. + // This defaults to `lua.RegistryGrowStep` + RegistryGrowStep int + // Controls whether or not libraries are opened by default + SkipOpenLibs bool + // Tells whether a Go stacktrace should be included in a Lua stacktrace when panics occur. + IncludeGoStackTrace bool + // If `MinimizeStackMemory` is set, the call stack will be automatically grown or shrank up to a limit of + // `CallStackSize` in order to minimize memory usage. This does incur a slight performance penalty. + MinimizeStackMemory bool +} + +/* }}} */ + +/* Debug {{{ */ + +type Debug struct { + frame *callFrame + Name string + What string + Source string + CurrentLine int + NUpvalues int + LineDefined int + LastLineDefined int +} + +/* }}} */ + +/* callFrame {{{ */ + +type callFrame struct { + Idx int + Fn *LFunction + Parent *callFrame + Pc int + Base int + LocalBase int + ReturnBase int + NArgs int + NRet int + TailCall int +} + +type callFrameStack interface { + Push(v callFrame) + Pop() *callFrame + Last() *callFrame + + SetSp(sp int) + Sp() int + At(sp int) *callFrame + + IsFull() bool + IsEmpty() bool + + FreeAll() +} + +type fixedCallFrameStack struct { + array []callFrame + sp int +} + +func newFixedCallFrameStack(size int) callFrameStack { + return &fixedCallFrameStack{ + array: make([]callFrame, size), + sp: 0, + } +} + +func (cs *fixedCallFrameStack) IsEmpty() bool { return cs.sp == 0 } + +func (cs *fixedCallFrameStack) IsFull() bool { return cs.sp == len(cs.array) } + +func (cs *fixedCallFrameStack) Clear() { + cs.sp = 0 +} + +func (cs *fixedCallFrameStack) Push(v callFrame) { + cs.array[cs.sp] = v + cs.array[cs.sp].Idx = cs.sp + cs.sp++ +} + +func (cs *fixedCallFrameStack) Sp() int { + return cs.sp +} + +func (cs *fixedCallFrameStack) SetSp(sp int) { + cs.sp = sp +} + +func (cs *fixedCallFrameStack) Last() *callFrame { + if cs.sp == 0 { + return nil + } + return &cs.array[cs.sp-1] +} + +func (cs *fixedCallFrameStack) At(sp int) *callFrame { + return &cs.array[sp] +} + +func (cs *fixedCallFrameStack) Pop() *callFrame { + cs.sp-- + return &cs.array[cs.sp] +} + +func (cs *fixedCallFrameStack) FreeAll() { + // nothing to do for fixed callframestack +} + +// FramesPerSegment should be a power of 2 constant for performance reasons. It will allow the go compiler to change +// the divs and mods into bitshifts. Max is 256 due to current use of uint8 to count how many frames in a segment are +// used. +const FramesPerSegment = 8 + +type callFrameStackSegment struct { + array [FramesPerSegment]callFrame +} +type segIdx uint16 +type autoGrowingCallFrameStack struct { + segments []*callFrameStackSegment + segIdx segIdx + // segSp is the number of frames in the current segment which are used. Full 'sp' value is segIdx * FramesPerSegment + segSp. + // It points to the next stack slot to use, so 0 means to use the 0th element in the segment, and a value of + // FramesPerSegment indicates that the segment is full and cannot accommodate another frame. + segSp uint8 +} + +var segmentPool sync.Pool + +func newCallFrameStackSegment() *callFrameStackSegment { + seg := segmentPool.Get() + if seg == nil { + return &callFrameStackSegment{} + } + return seg.(*callFrameStackSegment) +} + +func freeCallFrameStackSegment(seg *callFrameStackSegment) { + segmentPool.Put(seg) +} + +// newCallFrameStack allocates a new stack for a lua state, which will auto grow up to a max size of at least maxSize. +// it will actually grow up to the next segment size multiple after maxSize, where the segment size is dictated by +// FramesPerSegment. +func newAutoGrowingCallFrameStack(maxSize int) callFrameStack { + cs := &autoGrowingCallFrameStack{ + segments: make([]*callFrameStackSegment, (maxSize+(FramesPerSegment-1))/FramesPerSegment), + segIdx: 0, + } + cs.segments[0] = newCallFrameStackSegment() + return cs +} + +func (cs *autoGrowingCallFrameStack) IsEmpty() bool { + return cs.segIdx == 0 && cs.segSp == 0 +} + +// IsFull returns true if the stack cannot receive any more stack pushes without overflowing +func (cs *autoGrowingCallFrameStack) IsFull() bool { + return int(cs.segIdx) == len(cs.segments) && cs.segSp >= FramesPerSegment +} + +func (cs *autoGrowingCallFrameStack) Clear() { + for i := segIdx(1); i <= cs.segIdx; i++ { + freeCallFrameStackSegment(cs.segments[i]) + cs.segments[i] = nil + } + cs.segIdx = 0 + cs.segSp = 0 +} + +func (cs *autoGrowingCallFrameStack) FreeAll() { + for i := segIdx(0); i <= cs.segIdx; i++ { + freeCallFrameStackSegment(cs.segments[i]) + cs.segments[i] = nil + } +} + +// Push pushes the passed callFrame onto the stack. it panics if the stack is full, caller should call IsFull() before +// invoking this to avoid this. +func (cs *autoGrowingCallFrameStack) Push(v callFrame) { + curSeg := cs.segments[cs.segIdx] + if cs.segSp >= FramesPerSegment { + // segment full, push new segment if allowed + if cs.segIdx < segIdx(len(cs.segments)-1) { + curSeg = newCallFrameStackSegment() + cs.segIdx++ + cs.segments[cs.segIdx] = curSeg + cs.segSp = 0 + } else { + panic("lua callstack overflow") + } + } + curSeg.array[cs.segSp] = v + curSeg.array[cs.segSp].Idx = int(cs.segSp) + FramesPerSegment*int(cs.segIdx) + cs.segSp++ +} + +// Sp retrieves the current stack depth, which is the number of frames currently pushed on the stack. +func (cs *autoGrowingCallFrameStack) Sp() int { + return int(cs.segSp) + int(cs.segIdx)*FramesPerSegment +} + +// SetSp can be used to rapidly unwind the stack, freeing all stack frames on the way. It should not be used to +// allocate new stack space, use Push() for that. +func (cs *autoGrowingCallFrameStack) SetSp(sp int) { + desiredSegIdx := segIdx(sp / FramesPerSegment) + desiredFramesInLastSeg := uint8(sp % FramesPerSegment) + for { + if cs.segIdx <= desiredSegIdx { + break + } + freeCallFrameStackSegment(cs.segments[cs.segIdx]) + cs.segments[cs.segIdx] = nil + cs.segIdx-- + } + cs.segSp = desiredFramesInLastSeg +} + +func (cs *autoGrowingCallFrameStack) Last() *callFrame { + curSeg := cs.segments[cs.segIdx] + segSp := cs.segSp + if segSp == 0 { + if cs.segIdx == 0 { + return nil + } + curSeg = cs.segments[cs.segIdx-1] + segSp = FramesPerSegment + } + return &curSeg.array[segSp-1] +} + +func (cs *autoGrowingCallFrameStack) At(sp int) *callFrame { + segIdx := segIdx(sp / FramesPerSegment) + frameIdx := uint8(sp % FramesPerSegment) + return &cs.segments[segIdx].array[frameIdx] +} + +// Pop pops off the most recent stack frame and returns it +func (cs *autoGrowingCallFrameStack) Pop() *callFrame { + curSeg := cs.segments[cs.segIdx] + if cs.segSp == 0 { + if cs.segIdx == 0 { + // stack empty + return nil + } + freeCallFrameStackSegment(curSeg) + cs.segments[cs.segIdx] = nil + cs.segIdx-- + cs.segSp = FramesPerSegment + curSeg = cs.segments[cs.segIdx] + } + cs.segSp-- + return &curSeg.array[cs.segSp] +} + +/* }}} */ + +/* registry {{{ */ + +type registryHandler interface { + registryOverflow() +} +type registry struct { + array []LValue + top int + growBy int + maxSize int + alloc *allocator + handler registryHandler +} + +func newRegistry(handler registryHandler, initialSize int, growBy int, maxSize int, alloc *allocator) *registry { + return ®istry{make([]LValue, initialSize), 0, growBy, maxSize, alloc, handler} +} + +func (rg *registry) checkSize(requiredSize int) { // +inline-start + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } +} // +inline-end + +func (rg *registry) resize(requiredSize int) { // +inline-start + newSize := requiredSize + rg.growBy // give some padding + if newSize > rg.maxSize { + newSize = rg.maxSize + } + if newSize < requiredSize { + rg.handler.registryOverflow() + return + } + rg.forceResize(newSize) +} // +inline-end + +func (rg *registry) forceResize(newSize int) { + newSlice := make([]LValue, newSize) + copy(newSlice, rg.array[:rg.top]) // should we copy the area beyond top? there shouldn't be any valid values there so it shouldn't be necessary. + rg.array = newSlice +} + +func (rg *registry) SetTop(topi int) { // +inline-start + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := topi + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + oldtopi := rg.top + rg.top = topi + for i := oldtopi; i < rg.top; i++ { + rg.array[i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + if rg.top < oldtopi { + nilRange := rg.array[rg.top:oldtopi] + for i := range nilRange { + nilRange[i] = nil + } + } + //for i := rg.top; i < oldtop; i++ { + // rg.array[i] = LNil + //} +} // +inline-end + +func (rg *registry) Top() int { + return rg.top +} + +func (rg *registry) Push(v LValue) { + newSize := rg.top + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[rg.top] = v + rg.top++ +} + +func (rg *registry) Pop() LValue { + v := rg.array[rg.top-1] + rg.array[rg.top-1] = LNil + rg.top-- + return v +} + +func (rg *registry) Get(reg int) LValue { + return rg.array[reg] +} + +// CopyRange will move a section of values from index `start` to index `regv` +// It will move `n` values. +// `limit` specifies the maximum end range that can be copied from. If it's set to -1, then it defaults to stopping at +// the top of the registry (values beyond the top are not initialized, so if specifying an alternative `limit` you should +// pass a value <= rg.top. +// If start+n is beyond the limit, then nil values will be copied to the destination slots. +// After the copy, the registry is truncated to be at the end of the copied range, ie the original of the copied values +// are nilled out. (So top will be regv+n) +// CopyRange should ideally be renamed to MoveRange. +func (rg *registry) CopyRange(regv, start, limit, n int) { // +inline-start + newSize := regv + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + if limit == -1 || limit > rg.top { + limit = rg.top + } + for i := 0; i < n; i++ { + srcIdx := start + i + if srcIdx >= limit || srcIdx < 0 { + rg.array[regv+i] = LNil + } else { + rg.array[regv+i] = rg.array[srcIdx] + } + } + + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regv + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } +} // +inline-end + +// FillNil fills the registry with nil values from regm to regm+n and then sets the registry top to regm+n +func (rg *registry) FillNil(regm, n int) { // +inline-start + newSize := regm + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := 0; i < n; i++ { + rg.array[regm+i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regm + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } +} // +inline-end + +func (rg *registry) Insert(value LValue, reg int) { + top := rg.Top() + if reg >= top { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + regi := reg + vali := value + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return + } + top-- + for ; top >= reg; top-- { + // FIXME consider using copy() here if Insert() is called enough + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + regi := top + 1 + vali := rg.Get(top) + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + regi := reg + vali := value + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } +} + +func (rg *registry) Set(regi int, vali LValue) { // +inline-start + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } +} // +inline-end + +func (rg *registry) SetNumber(regi int, vali LNumber) { // +inline-start + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } +} // +inline-end + +func (rg *registry) IsFull() bool { + return rg.top >= cap(rg.array) +} + +/* }}} */ + +/* Global {{{ */ + +func newGlobal() *Global { + return &Global{ + MainThread: nil, + Registry: newLTable(0, 32), + Global: newLTable(0, 64), + builtinMts: make(map[int]LValue), + tempFiles: make([]*os.File, 0, 10), + } +} + +/* }}} */ + +/* package local methods {{{ */ + +func panicWithTraceback(L *LState) { + err := newApiError(ApiErrorRun, L.Get(-1)) + err.StackTrace = L.stackTrace(0) + panic(err) +} + +func panicWithoutTraceback(L *LState) { + err := newApiError(ApiErrorRun, L.Get(-1)) + panic(err) +} + +func newLState(options Options) *LState { + al := newAllocator(32) + ls := &LState{ + G: newGlobal(), + Parent: nil, + Panic: panicWithTraceback, + Dead: false, + Options: options, + + stop: 0, + alloc: al, + currentFrame: nil, + wrapped: false, + uvcache: nil, + hasErrorFunc: false, + mainLoop: mainLoop, + ctx: nil, + } + if options.MinimizeStackMemory { + ls.stack = newAutoGrowingCallFrameStack(options.CallStackSize) + } else { + ls.stack = newFixedCallFrameStack(options.CallStackSize) + } + ls.reg = newRegistry(ls, options.RegistrySize, options.RegistryGrowStep, options.RegistryMaxSize, al) + ls.Env = ls.G.Global + return ls +} + +func (ls *LState) printReg() { + println("-------------------------") + println("thread:", ls) + println("top:", ls.reg.Top()) + if ls.currentFrame != nil { + println("function base:", ls.currentFrame.Base) + println("return base:", ls.currentFrame.ReturnBase) + } else { + println("(vm not started)") + } + println("local base:", ls.currentLocalBase()) + for i := 0; i < ls.reg.Top(); i++ { + println(i, ls.reg.Get(i).String()) + } + println("-------------------------") +} + +func (ls *LState) printCallStack() { + println("-------------------------") + for i := 0; i < ls.stack.Sp(); i++ { + print(i) + print(" ") + frame := ls.stack.At(i) + if frame == nil { + break + } + if frame.Fn.IsG { + println("IsG:", true, "Frame:", frame, "Fn:", frame.Fn) + } else { + println("IsG:", false, "Frame:", frame, "Fn:", frame.Fn, "pc:", frame.Pc) + } + } + println("-------------------------") +} + +func (ls *LState) closeAllUpvalues() { // +inline-start + for cf := ls.currentFrame; cf != nil; cf = cf.Parent { + if !cf.Fn.IsG { + ls.closeUpvalues(cf.LocalBase) + } + } +} // +inline-end + +func (ls *LState) raiseError(level int, format string, args ...interface{}) { + if !ls.hasErrorFunc { + ls.closeAllUpvalues() + } + message := format + if len(args) > 0 { + message = fmt.Sprintf(format, args...) + } + if level > 0 { + message = fmt.Sprintf("%v %v", ls.where(level-1, true), message) + } + if ls.reg.IsFull() { + // if the registry is full then it won't be possible to push a value, in this case, force a larger size + ls.reg.forceResize(ls.reg.Top() + 1) + } + ls.reg.Push(LString(message)) + ls.Panic(ls) +} + +func (ls *LState) findLocal(frame *callFrame, no int) string { + fn := frame.Fn + if !fn.IsG { + if name, ok := fn.LocalName(no, frame.Pc-1); ok { + return name + } + } + var top int + if ls.currentFrame == frame { + top = ls.reg.Top() + } else if frame.Idx+1 < ls.stack.Sp() { + top = ls.stack.At(frame.Idx + 1).Base + } else { + return "" + } + if top-frame.LocalBase >= no { + return "(*temporary)" + } + return "" +} + +func (ls *LState) where(level int, skipg bool) string { + dbg, ok := ls.GetStack(level) + if !ok { + return "" + } + cf := dbg.frame + proto := cf.Fn.Proto + sourcename := "[G]" + if proto != nil { + sourcename = proto.SourceName + } else if skipg { + return ls.where(level+1, skipg) + } + line := "" + if proto != nil { + line = fmt.Sprintf("%v:", proto.DbgSourcePositions[cf.Pc-1]) + } + return fmt.Sprintf("%v:%v", sourcename, line) +} + +func (ls *LState) stackTrace(level int) string { + buf := []string{} + header := "stack traceback:" + if ls.currentFrame != nil { + i := 0 + for dbg, ok := ls.GetStack(i); ok; dbg, ok = ls.GetStack(i) { + cf := dbg.frame + buf = append(buf, fmt.Sprintf("\t%v in %v", ls.Where(i), ls.formattedFrameFuncName(cf))) + if !cf.Fn.IsG && cf.TailCall > 0 { + for tc := cf.TailCall; tc > 0; tc-- { + buf = append(buf, "\t(tailcall): ?") + i++ + } + } + i++ + } + } + buf = append(buf, fmt.Sprintf("\t%v: %v", "[G]", "?")) + buf = buf[intMax(0, intMin(level, len(buf))):len(buf)] + if len(buf) > 20 { + newbuf := make([]string, 0, 20) + newbuf = append(newbuf, buf[0:7]...) + newbuf = append(newbuf, "\t...") + newbuf = append(newbuf, buf[len(buf)-7:len(buf)]...) + buf = newbuf + } + return fmt.Sprintf("%s\n%s", header, strings.Join(buf, "\n")) +} + +func (ls *LState) formattedFrameFuncName(fr *callFrame) string { + name, ischunk := ls.frameFuncName(fr) + if ischunk { + return name + } + if name[0] != '(' && name[0] != '<' { + return fmt.Sprintf("function '%s'", name) + } + return fmt.Sprintf("function %s", name) +} + +func (ls *LState) rawFrameFuncName(fr *callFrame) string { + name, _ := ls.frameFuncName(fr) + return name +} + +func (ls *LState) frameFuncName(fr *callFrame) (string, bool) { + frame := fr.Parent + if frame == nil { + if ls.Parent == nil { + return "main chunk", true + } else { + return "corountine", true + } + } + if !frame.Fn.IsG { + pc := frame.Pc - 1 + for _, call := range frame.Fn.Proto.DbgCalls { + if call.Pc == pc { + name := call.Name + if (name == "?" || fr.TailCall > 0) && !fr.Fn.IsG { + name = fmt.Sprintf("<%v:%v>", fr.Fn.Proto.SourceName, fr.Fn.Proto.LineDefined) + } + return name, false + } + } + } + if !fr.Fn.IsG { + return fmt.Sprintf("<%v:%v>", fr.Fn.Proto.SourceName, fr.Fn.Proto.LineDefined), false + } + return "(anonymous)", false +} + +func (ls *LState) isStarted() bool { + return ls.currentFrame != nil +} + +func (ls *LState) kill() { + ls.Dead = true + if ls.ctxCancelFn != nil { + ls.ctxCancelFn() + } +} + +func (ls *LState) indexToReg(idx int) int { + base := ls.currentLocalBase() + if idx > 0 { + return base + idx - 1 + } else if idx == 0 { + return -1 + } else { + tidx := ls.reg.Top() + idx + if tidx < base { + return -1 + } + return tidx + } +} + +func (ls *LState) currentLocalBase() int { + base := 0 + if ls.currentFrame != nil { + base = ls.currentFrame.LocalBase + } + return base +} + +func (ls *LState) currentEnv() *LTable { + return ls.Env + /* + if ls.currentFrame == nil { + return ls.Env + } + return ls.currentFrame.Fn.Env + */ +} + +func (ls *LState) rkValue(idx int) LValue { + /* + if OpIsK(idx) { + return ls.currentFrame.Fn.Proto.Constants[opIndexK(idx)] + } + return ls.reg.Get(ls.currentFrame.LocalBase + idx) + */ + if (idx & opBitRk) != 0 { + return ls.currentFrame.Fn.Proto.Constants[idx & ^opBitRk] + } + return ls.reg.array[ls.currentFrame.LocalBase+idx] +} + +func (ls *LState) rkString(idx int) string { + if (idx & opBitRk) != 0 { + return ls.currentFrame.Fn.Proto.stringConstants[idx & ^opBitRk] + } + return string(ls.reg.array[ls.currentFrame.LocalBase+idx].(LString)) +} + +func (ls *LState) closeUpvalues(idx int) { // +inline-start + if ls.uvcache != nil { + var prev *Upvalue + for uv := ls.uvcache; uv != nil; uv = uv.next { + if uv.index >= idx { + if prev != nil { + prev.next = nil + } else { + ls.uvcache = nil + } + uv.Close() + } + prev = uv + } + } +} // +inline-end + +func (ls *LState) findUpvalue(idx int) *Upvalue { + var prev *Upvalue + var next *Upvalue + if ls.uvcache != nil { + for uv := ls.uvcache; uv != nil; uv = uv.next { + if uv.index == idx { + return uv + } + if uv.index > idx { + next = uv + break + } + prev = uv + } + } + uv := &Upvalue{reg: ls.reg, index: idx, closed: false} + if prev != nil { + prev.next = uv + } else { + ls.uvcache = uv + } + if next != nil { + uv.next = next + } + return uv +} + +func (ls *LState) metatable(lvalue LValue, rawget bool) LValue { + var metatable LValue = LNil + switch obj := lvalue.(type) { + case *LTable: + metatable = obj.Metatable + case *LUserData: + metatable = obj.Metatable + default: + if table, ok := ls.G.builtinMts[int(obj.Type())]; ok { + metatable = table + } + } + + if !rawget && metatable != LNil { + oldmt := metatable + if tb, ok := metatable.(*LTable); ok { + metatable = tb.RawGetString("__metatable") + if metatable == LNil { + metatable = oldmt + } + } + } + + return metatable +} + +func (ls *LState) metaOp1(lvalue LValue, event string) LValue { + if mt := ls.metatable(lvalue, true); mt != LNil { + if tb, ok := mt.(*LTable); ok { + return tb.RawGetString(event) + } + } + return LNil +} + +func (ls *LState) metaOp2(value1, value2 LValue, event string) LValue { + if mt := ls.metatable(value1, true); mt != LNil { + if tb, ok := mt.(*LTable); ok { + if ret := tb.RawGetString(event); ret != LNil { + return ret + } + } + } + if mt := ls.metatable(value2, true); mt != LNil { + if tb, ok := mt.(*LTable); ok { + return tb.RawGetString(event) + } + } + return LNil +} + +func (ls *LState) metaCall(lvalue LValue) (*LFunction, bool) { + if fn, ok := lvalue.(*LFunction); ok { + return fn, false + } + if fn, ok := ls.metaOp1(lvalue, "__call").(*LFunction); ok { + return fn, true + } + return nil, false +} + +func (ls *LState) initCallFrame(cf *callFrame) { // +inline-start + if cf.Fn.IsG { + ls.reg.SetTop(cf.LocalBase + cf.NArgs) + } else { + proto := cf.Fn.Proto + nargs := cf.NArgs + np := int(proto.NumParameters) + if nargs < np { + // default any missing arguments to nil + newSize := cf.LocalBase + np + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + rg := ls.reg + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := nargs; i < np; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + nargs = np + ls.reg.top = newSize + } + + if (proto.IsVarArg & VarArgIsVarArg) == 0 { + if nargs < int(proto.NumUsedRegisters) { + nargs = int(proto.NumUsedRegisters) + } + newSize := cf.LocalBase + nargs + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + rg := ls.reg + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := np; i < nargs; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + ls.reg.top = cf.LocalBase + int(proto.NumUsedRegisters) + } else { + /* swap vararg positions: + closure + namedparam1 <- lbase + namedparam2 + vararg1 + vararg2 + + TO + + closure + nil + nil + vararg1 + vararg2 + namedparam1 <- lbase + namedparam2 + */ + nvarargs := nargs - np + if nvarargs < 0 { + nvarargs = 0 + } + + ls.reg.SetTop(cf.LocalBase + nargs + np) + for i := 0; i < np; i++ { + //ls.reg.Set(cf.LocalBase+nargs+i, ls.reg.Get(cf.LocalBase+i)) + ls.reg.array[cf.LocalBase+nargs+i] = ls.reg.array[cf.LocalBase+i] + //ls.reg.Set(cf.LocalBase+i, LNil) + ls.reg.array[cf.LocalBase+i] = LNil + } + + if CompatVarArg { + ls.reg.SetTop(cf.LocalBase + nargs + np + 1) + if (proto.IsVarArg & VarArgNeedsArg) != 0 { + argtb := newLTable(nvarargs, 0) + for i := 0; i < nvarargs; i++ { + argtb.RawSetInt(i+1, ls.reg.Get(cf.LocalBase+np+i)) + } + argtb.RawSetString("n", LNumber(nvarargs)) + //ls.reg.Set(cf.LocalBase+nargs+np, argtb) + ls.reg.array[cf.LocalBase+nargs+np] = argtb + } else { + ls.reg.array[cf.LocalBase+nargs+np] = LNil + } + } + cf.LocalBase += nargs + maxreg := cf.LocalBase + int(proto.NumUsedRegisters) + ls.reg.SetTop(maxreg) + } + } +} // +inline-end + +func (ls *LState) pushCallFrame(cf callFrame, fn LValue, meta bool) { // +inline-start + if meta { + cf.NArgs++ + ls.reg.Insert(fn, cf.LocalBase) + } + if cf.Fn == nil { + ls.RaiseError("attempt to call a non-function object") + } + if ls.stack.IsFull() { + ls.RaiseError("stack overflow") + } + ls.stack.Push(cf) + newcf := ls.stack.Last() + // this section is inlined by go-inline + // source function is 'func (ls *LState) initCallFrame(cf *callFrame) ' in '_state.go' + { + cf := newcf + if cf.Fn.IsG { + ls.reg.SetTop(cf.LocalBase + cf.NArgs) + } else { + proto := cf.Fn.Proto + nargs := cf.NArgs + np := int(proto.NumParameters) + if nargs < np { + // default any missing arguments to nil + newSize := cf.LocalBase + np + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + rg := ls.reg + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := nargs; i < np; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + nargs = np + ls.reg.top = newSize + } + + if (proto.IsVarArg & VarArgIsVarArg) == 0 { + if nargs < int(proto.NumUsedRegisters) { + nargs = int(proto.NumUsedRegisters) + } + newSize := cf.LocalBase + nargs + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + rg := ls.reg + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := np; i < nargs; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + ls.reg.top = cf.LocalBase + int(proto.NumUsedRegisters) + } else { + /* swap vararg positions: + closure + namedparam1 <- lbase + namedparam2 + vararg1 + vararg2 + + TO + + closure + nil + nil + vararg1 + vararg2 + namedparam1 <- lbase + namedparam2 + */ + nvarargs := nargs - np + if nvarargs < 0 { + nvarargs = 0 + } + + ls.reg.SetTop(cf.LocalBase + nargs + np) + for i := 0; i < np; i++ { + //ls.reg.Set(cf.LocalBase+nargs+i, ls.reg.Get(cf.LocalBase+i)) + ls.reg.array[cf.LocalBase+nargs+i] = ls.reg.array[cf.LocalBase+i] + //ls.reg.Set(cf.LocalBase+i, LNil) + ls.reg.array[cf.LocalBase+i] = LNil + } + + if CompatVarArg { + ls.reg.SetTop(cf.LocalBase + nargs + np + 1) + if (proto.IsVarArg & VarArgNeedsArg) != 0 { + argtb := newLTable(nvarargs, 0) + for i := 0; i < nvarargs; i++ { + argtb.RawSetInt(i+1, ls.reg.Get(cf.LocalBase+np+i)) + } + argtb.RawSetString("n", LNumber(nvarargs)) + //ls.reg.Set(cf.LocalBase+nargs+np, argtb) + ls.reg.array[cf.LocalBase+nargs+np] = argtb + } else { + ls.reg.array[cf.LocalBase+nargs+np] = LNil + } + } + cf.LocalBase += nargs + maxreg := cf.LocalBase + int(proto.NumUsedRegisters) + ls.reg.SetTop(maxreg) + } + } + } + ls.currentFrame = newcf +} // +inline-end + +func (ls *LState) callR(nargs, nret, rbase int) { + base := ls.reg.Top() - nargs - 1 + if rbase < 0 { + rbase = base + } + lv := ls.reg.Get(base) + fn, meta := ls.metaCall(lv) + ls.pushCallFrame(callFrame{ + Fn: fn, + Pc: 0, + Base: base, + LocalBase: base + 1, + ReturnBase: rbase, + NArgs: nargs, + NRet: nret, + Parent: ls.currentFrame, + TailCall: 0, + }, lv, meta) + if ls.G.MainThread == nil { + ls.G.MainThread = ls + ls.G.CurrentThread = ls + ls.mainLoop(ls, nil) + } else { + ls.mainLoop(ls, ls.currentFrame) + } + if nret != MultRet { + ls.reg.SetTop(rbase + nret) + } +} + +func (ls *LState) getField(obj LValue, key LValue) LValue { + curobj := obj + for i := 0; i < MaxTableGetLoop; i++ { + tb, istable := curobj.(*LTable) + if istable { + ret := tb.RawGet(key) + if ret != LNil { + return ret + } + } + metaindex := ls.metaOp1(curobj, "__index") + if metaindex == LNil { + if !istable { + ls.RaiseError("attempt to index a non-table object(%v) with key '%s'", curobj.Type().String(), key.String()) + } + return LNil + } + if metaindex.Type() == LTFunction { + ls.reg.Push(metaindex) + ls.reg.Push(curobj) + ls.reg.Push(key) + ls.Call(2, 1) + return ls.reg.Pop() + } else { + curobj = metaindex + } + } + ls.RaiseError("too many recursions in gettable") + return nil +} + +func (ls *LState) getFieldString(obj LValue, key string) LValue { + curobj := obj + for i := 0; i < MaxTableGetLoop; i++ { + tb, istable := curobj.(*LTable) + if istable { + ret := tb.RawGetString(key) + if ret != LNil { + return ret + } + } + metaindex := ls.metaOp1(curobj, "__index") + if metaindex == LNil { + if !istable { + ls.RaiseError("attempt to index a non-table object(%v) with key '%s'", curobj.Type().String(), key) + } + return LNil + } + if metaindex.Type() == LTFunction { + ls.reg.Push(metaindex) + ls.reg.Push(curobj) + ls.reg.Push(LString(key)) + ls.Call(2, 1) + return ls.reg.Pop() + } else { + curobj = metaindex + } + } + ls.RaiseError("too many recursions in gettable") + return nil +} + +func (ls *LState) setField(obj LValue, key LValue, value LValue) { + curobj := obj + for i := 0; i < MaxTableGetLoop; i++ { + tb, istable := curobj.(*LTable) + if istable { + if tb.RawGet(key) != LNil { + ls.RawSet(tb, key, value) + return + } + } + metaindex := ls.metaOp1(curobj, "__newindex") + if metaindex == LNil { + if !istable { + ls.RaiseError("attempt to index a non-table object(%v) with key '%s'", curobj.Type().String(), key.String()) + } + ls.RawSet(tb, key, value) + return + } + if metaindex.Type() == LTFunction { + ls.reg.Push(metaindex) + ls.reg.Push(curobj) + ls.reg.Push(key) + ls.reg.Push(value) + ls.Call(3, 0) + return + } else { + curobj = metaindex + } + } + ls.RaiseError("too many recursions in settable") +} + +func (ls *LState) setFieldString(obj LValue, key string, value LValue) { + curobj := obj + for i := 0; i < MaxTableGetLoop; i++ { + tb, istable := curobj.(*LTable) + if istable { + if tb.RawGetString(key) != LNil { + tb.RawSetString(key, value) + return + } + } + metaindex := ls.metaOp1(curobj, "__newindex") + if metaindex == LNil { + if !istable { + ls.RaiseError("attempt to index a non-table object(%v) with key '%s'", curobj.Type().String(), key) + } + tb.RawSetString(key, value) + return + } + if metaindex.Type() == LTFunction { + ls.reg.Push(metaindex) + ls.reg.Push(curobj) + ls.reg.Push(LString(key)) + ls.reg.Push(value) + ls.Call(3, 0) + return + } else { + curobj = metaindex + } + } + ls.RaiseError("too many recursions in settable") +} + +/* }}} */ + +/* api methods {{{ */ + +func NewState(opts ...Options) *LState { + var ls *LState + if len(opts) == 0 { + ls = newLState(Options{ + CallStackSize: CallStackSize, + RegistrySize: RegistrySize, + }) + ls.OpenLibs() + } else { + if opts[0].CallStackSize < 1 { + opts[0].CallStackSize = CallStackSize + } + if opts[0].RegistrySize < 128 { + opts[0].RegistrySize = RegistrySize + } + if opts[0].RegistryMaxSize < opts[0].RegistrySize { + opts[0].RegistryMaxSize = 0 // disable growth if max size is smaller than initial size + } else { + // if growth enabled, grow step is set + if opts[0].RegistryGrowStep < 1 { + opts[0].RegistryGrowStep = RegistryGrowStep + } + } + ls = newLState(opts[0]) + if !opts[0].SkipOpenLibs { + ls.OpenLibs() + } + } + return ls +} + +func (ls *LState) IsClosed() bool { + return ls.stack == nil +} + +func (ls *LState) Close() { + atomic.AddInt32(&ls.stop, 1) + for _, file := range ls.G.tempFiles { + // ignore errors in these operations + file.Close() + os.Remove(file.Name()) + } + ls.stack.FreeAll() + ls.stack = nil +} + +/* registry operations {{{ */ + +func (ls *LState) GetTop() int { + return ls.reg.Top() - ls.currentLocalBase() +} + +func (ls *LState) SetTop(idx int) { + base := ls.currentLocalBase() + newtop := ls.indexToReg(idx) + 1 + if newtop < base { + ls.reg.SetTop(base) + } else { + ls.reg.SetTop(newtop) + } +} + +func (ls *LState) Replace(idx int, value LValue) { + base := ls.currentLocalBase() + if idx > 0 { + reg := base + idx - 1 + if reg < ls.reg.Top() { + ls.reg.Set(reg, value) + } + } else if idx == 0 { + } else if idx > RegistryIndex { + if tidx := ls.reg.Top() + idx; tidx >= base { + ls.reg.Set(tidx, value) + } + } else { + switch idx { + case RegistryIndex: + if tb, ok := value.(*LTable); ok { + ls.G.Registry = tb + } else { + ls.RaiseError("registry must be a table(%v)", value.Type().String()) + } + case EnvironIndex: + if ls.currentFrame == nil { + ls.RaiseError("no calling environment") + } + if tb, ok := value.(*LTable); ok { + ls.currentFrame.Fn.Env = tb + } else { + ls.RaiseError("environment must be a table(%v)", value.Type().String()) + } + case GlobalsIndex: + if tb, ok := value.(*LTable); ok { + ls.G.Global = tb + } else { + ls.RaiseError("_G must be a table(%v)", value.Type().String()) + } + default: + fn := ls.currentFrame.Fn + index := GlobalsIndex - idx - 1 + if index < len(fn.Upvalues) { + fn.Upvalues[index].SetValue(value) + } + } + } +} + +func (ls *LState) Get(idx int) LValue { + base := ls.currentLocalBase() + if idx > 0 { + reg := base + idx - 1 + if reg < ls.reg.Top() { + return ls.reg.Get(reg) + } + return LNil + } else if idx == 0 { + return LNil + } else if idx > RegistryIndex { + tidx := ls.reg.Top() + idx + if tidx < base { + return LNil + } + return ls.reg.Get(tidx) + } else { + switch idx { + case RegistryIndex: + return ls.G.Registry + case EnvironIndex: + if ls.currentFrame == nil { + return ls.Env + } + return ls.currentFrame.Fn.Env + case GlobalsIndex: + return ls.G.Global + default: + fn := ls.currentFrame.Fn + index := GlobalsIndex - idx - 1 + if index < len(fn.Upvalues) { + return fn.Upvalues[index].Value() + } + return LNil + } + } + return LNil +} + +func (ls *LState) Push(value LValue) { + ls.reg.Push(value) +} + +func (ls *LState) Pop(n int) { + for i := 0; i < n; i++ { + if ls.GetTop() == 0 { + ls.RaiseError("register underflow") + } + ls.reg.Pop() + } +} + +func (ls *LState) Insert(value LValue, index int) { + reg := ls.indexToReg(index) + top := ls.reg.Top() + if reg >= top { + ls.reg.Set(reg, value) + return + } + if reg <= ls.currentLocalBase() { + reg = ls.currentLocalBase() + } + top-- + for ; top >= reg; top-- { + ls.reg.Set(top+1, ls.reg.Get(top)) + } + ls.reg.Set(reg, value) +} + +func (ls *LState) Remove(index int) { + reg := ls.indexToReg(index) + top := ls.reg.Top() + switch { + case reg >= top: + return + case reg < ls.currentLocalBase(): + return + case reg == top-1: + ls.Pop(1) + return + } + for i := reg; i < top-1; i++ { + ls.reg.Set(i, ls.reg.Get(i+1)) + } + ls.reg.SetTop(top - 1) +} + +/* }}} */ + +/* object allocation {{{ */ + +func (ls *LState) NewTable() *LTable { + return newLTable(defaultArrayCap, defaultHashCap) +} + +func (ls *LState) CreateTable(acap, hcap int) *LTable { + return newLTable(acap, hcap) +} + +// NewThread returns a new LState that shares with the original state all global objects. +// If the original state has context.Context, the new state has a new child context of the original state and this function returns its cancel function. +func (ls *LState) NewThread() (*LState, context.CancelFunc) { + thread := newLState(ls.Options) + thread.G = ls.G + thread.Env = ls.Env + var f context.CancelFunc = nil + if ls.ctx != nil { + thread.mainLoop = mainLoopWithContext + thread.ctx, f = context.WithCancel(ls.ctx) + thread.ctxCancelFn = f + } + return thread, f +} + +func (ls *LState) NewFunctionFromProto(proto *FunctionProto) *LFunction { + return newLFunctionL(proto, ls.Env, int(proto.NumUpvalues)) +} + +func (ls *LState) NewUserData() *LUserData { + return &LUserData{ + Env: ls.currentEnv(), + Metatable: LNil, + } +} + +func (ls *LState) NewFunction(fn LGFunction) *LFunction { + return newLFunctionG(fn, ls.currentEnv(), 0) +} + +func (ls *LState) NewClosure(fn LGFunction, upvalues ...LValue) *LFunction { + cl := newLFunctionG(fn, ls.currentEnv(), len(upvalues)) + for i, lv := range upvalues { + cl.Upvalues[i] = &Upvalue{} + cl.Upvalues[i].Close() + cl.Upvalues[i].SetValue(lv) + } + return cl +} + +/* }}} */ + +/* toType {{{ */ + +func (ls *LState) ToBool(n int) bool { + return LVAsBool(ls.Get(n)) +} + +func (ls *LState) ToInt(n int) int { + if lv, ok := ls.Get(n).(LNumber); ok { + return int(lv) + } + if lv, ok := ls.Get(n).(LString); ok { + if num, err := parseNumber(string(lv)); err == nil { + return int(num) + } + } + return 0 +} + +func (ls *LState) ToInt64(n int) int64 { + if lv, ok := ls.Get(n).(LNumber); ok { + return int64(lv) + } + if lv, ok := ls.Get(n).(LString); ok { + if num, err := parseNumber(string(lv)); err == nil { + return int64(num) + } + } + return 0 +} + +func (ls *LState) ToNumber(n int) LNumber { + return LVAsNumber(ls.Get(n)) +} + +func (ls *LState) ToString(n int) string { + return LVAsString(ls.Get(n)) +} + +func (ls *LState) ToTable(n int) *LTable { + if lv, ok := ls.Get(n).(*LTable); ok { + return lv + } + return nil +} + +func (ls *LState) ToFunction(n int) *LFunction { + if lv, ok := ls.Get(n).(*LFunction); ok { + return lv + } + return nil +} + +func (ls *LState) ToUserData(n int) *LUserData { + if lv, ok := ls.Get(n).(*LUserData); ok { + return lv + } + return nil +} + +func (ls *LState) ToThread(n int) *LState { + if lv, ok := ls.Get(n).(*LState); ok { + return lv + } + return nil +} + +/* }}} */ + +/* error & debug operations {{{ */ + +func (ls *LState) registryOverflow() { + ls.RaiseError("registry overflow") +} + +// This function is equivalent to luaL_error( http://www.lua.org/manual/5.1/manual.html#luaL_error ). +func (ls *LState) RaiseError(format string, args ...interface{}) { + ls.raiseError(1, format, args...) +} + +// This function is equivalent to lua_error( http://www.lua.org/manual/5.1/manual.html#lua_error ). +func (ls *LState) Error(lv LValue, level int) { + if str, ok := lv.(LString); ok { + ls.raiseError(level, string(str)) + } else { + if !ls.hasErrorFunc { + ls.closeAllUpvalues() + } + ls.Push(lv) + ls.Panic(ls) + } +} + +func (ls *LState) GetInfo(what string, dbg *Debug, fn LValue) (LValue, error) { + if !strings.HasPrefix(what, ">") { + fn = dbg.frame.Fn + } else { + what = what[1:] + } + f, ok := fn.(*LFunction) + if !ok { + return LNil, newApiErrorS(ApiErrorRun, "can not get debug info(an object in not a function)") + } + + retfn := false + for _, c := range what { + switch c { + case 'f': + retfn = true + case 'S': + if dbg.frame != nil && dbg.frame.Parent == nil { + dbg.What = "main" + } else if f.IsG { + dbg.What = "G" + } else if dbg.frame != nil && dbg.frame.TailCall > 0 { + dbg.What = "tail" + } else { + dbg.What = "Lua" + } + if !f.IsG { + dbg.Source = f.Proto.SourceName + dbg.LineDefined = f.Proto.LineDefined + dbg.LastLineDefined = f.Proto.LastLineDefined + } + case 'l': + if !f.IsG && dbg.frame != nil { + if dbg.frame.Pc > 0 { + dbg.CurrentLine = f.Proto.DbgSourcePositions[dbg.frame.Pc-1] + } + } else { + dbg.CurrentLine = -1 + } + case 'u': + dbg.NUpvalues = len(f.Upvalues) + case 'n': + if dbg.frame != nil { + dbg.Name = ls.rawFrameFuncName(dbg.frame) + } + default: + return LNil, newApiErrorS(ApiErrorRun, "invalid what: "+string(c)) + } + } + + if retfn { + return f, nil + } + return LNil, nil + +} + +func (ls *LState) GetStack(level int) (*Debug, bool) { + frame := ls.currentFrame + for ; level > 0 && frame != nil; frame = frame.Parent { + level-- + if !frame.Fn.IsG { + level -= frame.TailCall + } + } + + if level == 0 && frame != nil { + return &Debug{frame: frame}, true + } else if level < 0 && ls.stack.Sp() > 0 { + return &Debug{frame: ls.stack.At(0)}, true + } + return &Debug{}, false +} + +func (ls *LState) GetLocal(dbg *Debug, no int) (string, LValue) { + frame := dbg.frame + if name := ls.findLocal(frame, no); len(name) > 0 { + return name, ls.reg.Get(frame.LocalBase + no - 1) + } + return "", LNil +} + +func (ls *LState) SetLocal(dbg *Debug, no int, lv LValue) string { + frame := dbg.frame + if name := ls.findLocal(frame, no); len(name) > 0 { + ls.reg.Set(frame.LocalBase+no-1, lv) + return name + } + return "" +} + +func (ls *LState) GetUpvalue(fn *LFunction, no int) (string, LValue) { + if fn.IsG { + return "", LNil + } + + no-- + if no >= 0 && no < len(fn.Upvalues) { + return fn.Proto.DbgUpvalues[no], fn.Upvalues[no].Value() + } + return "", LNil +} + +func (ls *LState) SetUpvalue(fn *LFunction, no int, lv LValue) string { + if fn.IsG { + return "" + } + + no-- + if no >= 0 && no < len(fn.Upvalues) { + fn.Upvalues[no].SetValue(lv) + return fn.Proto.DbgUpvalues[no] + } + return "" +} + +/* }}} */ + +/* env operations {{{ */ + +func (ls *LState) GetFEnv(obj LValue) LValue { + switch lv := obj.(type) { + case *LFunction: + return lv.Env + case *LUserData: + return lv.Env + case *LState: + return lv.Env + } + return LNil +} + +func (ls *LState) SetFEnv(obj LValue, env LValue) { + tb, ok := env.(*LTable) + if !ok { + ls.RaiseError("cannot use %v as an environment", env.Type().String()) + } + + switch lv := obj.(type) { + case *LFunction: + lv.Env = tb + case *LUserData: + lv.Env = tb + case *LState: + lv.Env = tb + } + /* do nothing */ +} + +/* }}} */ + +/* table operations {{{ */ + +func (ls *LState) RawGet(tb *LTable, key LValue) LValue { + return tb.RawGet(key) +} + +func (ls *LState) RawGetInt(tb *LTable, key int) LValue { + return tb.RawGetInt(key) +} + +func (ls *LState) GetField(obj LValue, skey string) LValue { + return ls.getFieldString(obj, skey) +} + +func (ls *LState) GetTable(obj LValue, key LValue) LValue { + return ls.getField(obj, key) +} + +func (ls *LState) RawSet(tb *LTable, key LValue, value LValue) { + if n, ok := key.(LNumber); ok && math.IsNaN(float64(n)) { + ls.RaiseError("table index is NaN") + } else if key == LNil { + ls.RaiseError("table index is nil") + } + tb.RawSet(key, value) +} + +func (ls *LState) RawSetInt(tb *LTable, key int, value LValue) { + tb.RawSetInt(key, value) +} + +func (ls *LState) SetField(obj LValue, key string, value LValue) { + ls.setFieldString(obj, key, value) +} + +func (ls *LState) SetTable(obj LValue, key LValue, value LValue) { + ls.setField(obj, key, value) +} + +func (ls *LState) ForEach(tb *LTable, cb func(LValue, LValue)) { + tb.ForEach(cb) +} + +func (ls *LState) GetGlobal(name string) LValue { + return ls.GetField(ls.Get(GlobalsIndex), name) +} + +func (ls *LState) SetGlobal(name string, value LValue) { + ls.SetField(ls.Get(GlobalsIndex), name, value) +} + +func (ls *LState) Next(tb *LTable, key LValue) (LValue, LValue) { + return tb.Next(key) +} + +/* }}} */ + +/* unary operations {{{ */ + +func (ls *LState) ObjLen(v1 LValue) int { + if v1.Type() == LTString { + return len(string(v1.(LString))) + } + op := ls.metaOp1(v1, "__len") + if op.Type() == LTFunction { + ls.Push(op) + ls.Push(v1) + ls.Call(1, 1) + ret := ls.reg.Pop() + if ret.Type() == LTNumber { + return int(ret.(LNumber)) + } + } else if v1.Type() == LTTable { + return v1.(*LTable).Len() + } + return 0 +} + +/* }}} */ + +/* binary operations {{{ */ + +func (ls *LState) Concat(values ...LValue) string { + top := ls.reg.Top() + for _, value := range values { + ls.reg.Push(value) + } + ret := stringConcat(ls, len(values), ls.reg.Top()-1) + ls.reg.SetTop(top) + return LVAsString(ret) +} + +func (ls *LState) LessThan(lhs, rhs LValue) bool { + return lessThan(ls, lhs, rhs) +} + +func (ls *LState) Equal(lhs, rhs LValue) bool { + return equals(ls, lhs, rhs, false) +} + +func (ls *LState) RawEqual(lhs, rhs LValue) bool { + return equals(ls, lhs, rhs, true) +} + +/* }}} */ + +/* register operations {{{ */ + +func (ls *LState) Register(name string, fn LGFunction) { + ls.SetGlobal(name, ls.NewFunction(fn)) +} + +/* }}} */ + +/* load and function call operations {{{ */ + +func (ls *LState) Load(reader io.Reader, name string) (*LFunction, error) { + chunk, err := parse.Parse(reader, name) + if err != nil { + return nil, newApiErrorE(ApiErrorSyntax, err) + } + proto, err := Compile(chunk, name) + if err != nil { + return nil, newApiErrorE(ApiErrorSyntax, err) + } + return newLFunctionL(proto, ls.currentEnv(), 0), nil +} + +func (ls *LState) Call(nargs, nret int) { + ls.callR(nargs, nret, -1) +} + +func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) { + err = nil + sp := ls.stack.Sp() + base := ls.reg.Top() - nargs - 1 + oldpanic := ls.Panic + ls.Panic = panicWithoutTraceback + if errfunc != nil { + ls.hasErrorFunc = true + } + defer func() { + ls.Panic = oldpanic + ls.hasErrorFunc = false + rcv := recover() + if rcv != nil { + if _, ok := rcv.(*ApiError); !ok { + err = newApiErrorS(ApiErrorPanic, fmt.Sprint(rcv)) + if ls.Options.IncludeGoStackTrace { + buf := make([]byte, 4096) + runtime.Stack(buf, false) + err.(*ApiError).StackTrace = strings.Trim(string(buf), "\000") + "\n" + ls.stackTrace(0) + } + } else { + err = rcv.(*ApiError) + } + if errfunc != nil { + ls.Push(errfunc) + ls.Push(err.(*ApiError).Object) + ls.Panic = panicWithoutTraceback + defer func() { + ls.Panic = oldpanic + rcv := recover() + if rcv != nil { + if _, ok := rcv.(*ApiError); !ok { + err = newApiErrorS(ApiErrorPanic, fmt.Sprint(rcv)) + if ls.Options.IncludeGoStackTrace { + buf := make([]byte, 4096) + runtime.Stack(buf, false) + err.(*ApiError).StackTrace = strings.Trim(string(buf), "\000") + ls.stackTrace(0) + } + } else { + err = rcv.(*ApiError) + err.(*ApiError).StackTrace = ls.stackTrace(0) + } + ls.stack.SetSp(sp) + ls.currentFrame = ls.stack.Last() + ls.reg.SetTop(base) + } + }() + ls.Call(1, 1) + err = newApiError(ApiErrorError, ls.Get(-1)) + } else if len(err.(*ApiError).StackTrace) == 0 { + err.(*ApiError).StackTrace = ls.stackTrace(0) + } + ls.stack.SetSp(sp) + ls.currentFrame = ls.stack.Last() + ls.reg.SetTop(base) + } + ls.stack.SetSp(sp) + if sp == 0 { + ls.currentFrame = nil + } + }() + + ls.Call(nargs, nret) + + return +} + +func (ls *LState) GPCall(fn LGFunction, data LValue) error { + ls.Push(newLFunctionG(fn, ls.currentEnv(), 0)) + ls.Push(data) + return ls.PCall(1, MultRet, nil) +} + +func (ls *LState) CallByParam(cp P, args ...LValue) error { + ls.Push(cp.Fn) + for _, arg := range args { + ls.Push(arg) + } + + if cp.Protect { + return ls.PCall(len(args), cp.NRet, cp.Handler) + } + ls.Call(len(args), cp.NRet) + return nil +} + +/* }}} */ + +/* metatable operations {{{ */ + +func (ls *LState) GetMetatable(obj LValue) LValue { + return ls.metatable(obj, false) +} + +func (ls *LState) SetMetatable(obj LValue, mt LValue) { + switch mt.(type) { + case *LNilType, *LTable: + default: + ls.RaiseError("metatable must be a table or nil, but got %v", mt.Type().String()) + } + + switch v := obj.(type) { + case *LTable: + v.Metatable = mt + case *LUserData: + v.Metatable = mt + default: + ls.G.builtinMts[int(obj.Type())] = mt + } +} + +/* }}} */ + +/* coroutine operations {{{ */ + +func (ls *LState) Status(th *LState) string { + status := "suspended" + if th.Dead { + status = "dead" + } else if ls.G.CurrentThread == th { + status = "running" + } else if ls.Parent == th { + status = "normal" + } + return status +} + +func (ls *LState) Resume(th *LState, fn *LFunction, args ...LValue) (ResumeState, error, []LValue) { + isstarted := th.isStarted() + if !isstarted { + base := 0 + th.stack.Push(callFrame{ + Fn: fn, + Pc: 0, + Base: base, + LocalBase: base + 1, + ReturnBase: base, + NArgs: 0, + NRet: MultRet, + Parent: nil, + TailCall: 0, + }) + } + + if ls.G.CurrentThread == th { + return ResumeError, newApiErrorS(ApiErrorRun, "can not resume a running thread"), nil + } + if th.Dead { + return ResumeError, newApiErrorS(ApiErrorRun, "can not resume a dead thread"), nil + } + th.Parent = ls + ls.G.CurrentThread = th + if !isstarted { + cf := th.stack.Last() + th.currentFrame = cf + th.SetTop(0) + for _, arg := range args { + th.Push(arg) + } + cf.NArgs = len(args) + th.initCallFrame(cf) + th.Panic = panicWithoutTraceback + } else { + for _, arg := range args { + th.Push(arg) + } + } + top := ls.GetTop() + threadRun(th) + haserror := LVIsFalse(ls.Get(top + 1)) + ret := make([]LValue, 0, ls.GetTop()) + for idx := top + 2; idx <= ls.GetTop(); idx++ { + ret = append(ret, ls.Get(idx)) + } + if len(ret) == 0 { + ret = append(ret, LNil) + } + ls.SetTop(top) + + if haserror { + return ResumeError, newApiError(ApiErrorRun, ret[0]), nil + } else if th.stack.IsEmpty() { + return ResumeOK, nil, ret + } + return ResumeYield, nil, ret +} + +func (ls *LState) Yield(values ...LValue) int { + ls.SetTop(0) + for _, lv := range values { + ls.Push(lv) + } + return -1 +} + +func (ls *LState) XMoveTo(other *LState, n int) { + if ls == other { + return + } + top := ls.GetTop() + n = intMin(n, top) + for i := n; i > 0; i-- { + other.Push(ls.Get(top - i + 1)) + } + ls.SetTop(top - n) +} + +/* }}} */ + +/* GopherLua original APIs {{{ */ + +// Set maximum memory size. This function can only be called from the main thread. +func (ls *LState) SetMx(mx int) { + if ls.Parent != nil { + ls.RaiseError("sub threads are not allowed to set a memory limit") + } + go func() { + limit := uint64(mx * 1024 * 1024) //MB + var s runtime.MemStats + for atomic.LoadInt32(&ls.stop) == 0 { + runtime.ReadMemStats(&s) + if s.Alloc >= limit { + fmt.Println("out of memory") + os.Exit(3) + } + time.Sleep(100 * time.Millisecond) + } + }() +} + +// SetContext set a context ctx to this LState. The provided ctx must be non-nil. +func (ls *LState) SetContext(ctx context.Context) { + ls.mainLoop = mainLoopWithContext + ls.ctx = ctx +} + +// Context returns the LState's context. To change the context, use WithContext. +func (ls *LState) Context() context.Context { + return ls.ctx +} + +// RemoveContext removes the context associated with this LState and returns this context. +func (ls *LState) RemoveContext() context.Context { + oldctx := ls.ctx + ls.mainLoop = mainLoop + ls.ctx = nil + return oldctx +} + +// Converts the Lua value at the given acceptable index to the chan LValue. +func (ls *LState) ToChannel(n int) chan LValue { + if lv, ok := ls.Get(n).(LChannel); ok { + return (chan LValue)(lv) + } + return nil +} + +// RemoveCallerFrame removes the stack frame above the current stack frame. This is useful in tail calls. It returns +// the new current frame. +func (ls *LState) RemoveCallerFrame() *callFrame { + cs := ls.stack + sp := cs.Sp() + parentFrame := cs.At(sp - 2) + currentFrame := cs.At(sp - 1) + parentsParentFrame := parentFrame.Parent + *parentFrame = *currentFrame + parentFrame.Parent = parentsParentFrame + parentFrame.Idx = sp - 2 + cs.Pop() + return parentFrame +} + +/* }}} */ + +/* }}} */ + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/stringlib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/stringlib.go new file mode 100644 index 00000000..f484c2b3 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/stringlib.go @@ -0,0 +1,448 @@ +package lua + +import ( + "fmt" + "strings" + + "github.com/yuin/gopher-lua/pm" +) + +const emptyLString LString = LString("") + +func OpenString(L *LState) int { + var mod *LTable + //_, ok := L.G.builtinMts[int(LTString)] + //if !ok { + mod = L.RegisterModule(StringLibName, strFuncs).(*LTable) + gmatch := L.NewClosure(strGmatch, L.NewFunction(strGmatchIter)) + mod.RawSetString("gmatch", gmatch) + mod.RawSetString("gfind", gmatch) + mod.RawSetString("__index", mod) + L.G.builtinMts[int(LTString)] = mod + //} + L.Push(mod) + return 1 +} + +var strFuncs = map[string]LGFunction{ + "byte": strByte, + "char": strChar, + "dump": strDump, + "find": strFind, + "format": strFormat, + "gsub": strGsub, + "len": strLen, + "lower": strLower, + "match": strMatch, + "rep": strRep, + "reverse": strReverse, + "sub": strSub, + "upper": strUpper, +} + +func strByte(L *LState) int { + str := L.CheckString(1) + start := L.OptInt(2, 1) - 1 + end := L.OptInt(3, -1) + l := len(str) + if start < 0 { + start = l + start + 1 + } + if end < 0 { + end = l + end + 1 + } + + if L.GetTop() == 2 { + if start < 0 || start >= l { + return 0 + } + L.Push(LNumber(str[start])) + return 1 + } + + start = intMax(start, 0) + end = intMin(end, l) + if end < 0 || end <= start || start >= l { + return 0 + } + + for i := start; i < end; i++ { + L.Push(LNumber(str[i])) + } + return end - start +} + +func strChar(L *LState) int { + top := L.GetTop() + bytes := make([]byte, L.GetTop()) + for i := 1; i <= top; i++ { + bytes[i-1] = uint8(L.CheckInt(i)) + } + L.Push(LString(string(bytes))) + return 1 +} + +func strDump(L *LState) int { + L.RaiseError("GopherLua does not support the string.dump") + return 0 +} + +func strFind(L *LState) int { + str := L.CheckString(1) + pattern := L.CheckString(2) + if len(pattern) == 0 { + L.Push(LNumber(1)) + L.Push(LNumber(0)) + return 2 + } + init := luaIndex2StringIndex(str, L.OptInt(3, 1), true) + plain := false + if L.GetTop() == 4 { + plain = LVAsBool(L.Get(4)) + } + + if plain { + pos := strings.Index(str[init:], pattern) + if pos < 0 { + L.Push(LNil) + return 1 + } + L.Push(LNumber(init+pos) + 1) + L.Push(LNumber(init + pos + len(pattern))) + return 2 + } + + mds, err := pm.Find(pattern, unsafeFastStringToReadOnlyBytes(str), init, 1) + if err != nil { + L.RaiseError(err.Error()) + } + if len(mds) == 0 { + L.Push(LNil) + return 1 + } + md := mds[0] + L.Push(LNumber(md.Capture(0) + 1)) + L.Push(LNumber(md.Capture(1))) + for i := 2; i < md.CaptureLength(); i += 2 { + if md.IsPosCapture(i) { + L.Push(LNumber(md.Capture(i))) + } else { + L.Push(LString(str[md.Capture(i):md.Capture(i+1)])) + } + } + return md.CaptureLength()/2 + 1 +} + +func strFormat(L *LState) int { + str := L.CheckString(1) + args := make([]interface{}, L.GetTop()-1) + top := L.GetTop() + for i := 2; i <= top; i++ { + args[i-2] = L.Get(i) + } + npat := strings.Count(str, "%") - strings.Count(str, "%%") + L.Push(LString(fmt.Sprintf(str, args[:intMin(npat, len(args))]...))) + return 1 +} + +func strGsub(L *LState) int { + str := L.CheckString(1) + pat := L.CheckString(2) + L.CheckTypes(3, LTString, LTTable, LTFunction) + repl := L.CheckAny(3) + limit := L.OptInt(4, -1) + + mds, err := pm.Find(pat, unsafeFastStringToReadOnlyBytes(str), 0, limit) + if err != nil { + L.RaiseError(err.Error()) + } + if len(mds) == 0 { + L.SetTop(1) + L.Push(LNumber(0)) + return 2 + } + switch lv := repl.(type) { + case LString: + L.Push(LString(strGsubStr(L, str, string(lv), mds))) + case *LTable: + L.Push(LString(strGsubTable(L, str, lv, mds))) + case *LFunction: + L.Push(LString(strGsubFunc(L, str, lv, mds))) + } + L.Push(LNumber(len(mds))) + return 2 +} + +type replaceInfo struct { + Indicies []int + String string +} + +func checkCaptureIndex(L *LState, m *pm.MatchData, idx int) { + if idx <= 2 { + return + } + if idx >= m.CaptureLength() { + L.RaiseError("invalid capture index") + } +} + +func capturedString(L *LState, m *pm.MatchData, str string, idx int) string { + checkCaptureIndex(L, m, idx) + if idx >= m.CaptureLength() && idx == 2 { + idx = 0 + } + if m.IsPosCapture(idx) { + return fmt.Sprint(m.Capture(idx)) + } else { + return str[m.Capture(idx):m.Capture(idx+1)] + } + +} + +func strGsubDoReplace(str string, info []replaceInfo) string { + offset := 0 + buf := []byte(str) + for _, replace := range info { + oldlen := len(buf) + b1 := append([]byte(""), buf[0:offset+replace.Indicies[0]]...) + b2 := []byte("") + index2 := offset + replace.Indicies[1] + if index2 <= len(buf) { + b2 = append(b2, buf[index2:len(buf)]...) + } + buf = append(b1, replace.String...) + buf = append(buf, b2...) + offset += len(buf) - oldlen + } + return string(buf) +} + +func strGsubStr(L *LState, str string, repl string, matches []*pm.MatchData) string { + infoList := make([]replaceInfo, 0, len(matches)) + for _, match := range matches { + start, end := match.Capture(0), match.Capture(1) + sc := newFlagScanner('%', "", "", repl) + for c, eos := sc.Next(); !eos; c, eos = sc.Next() { + if !sc.ChangeFlag { + if sc.HasFlag { + if c >= '0' && c <= '9' { + sc.AppendString(capturedString(L, match, str, 2*(int(c)-48))) + } else { + sc.AppendChar('%') + sc.AppendChar(c) + } + sc.HasFlag = false + } else { + sc.AppendChar(c) + } + } + } + infoList = append(infoList, replaceInfo{[]int{start, end}, sc.String()}) + } + + return strGsubDoReplace(str, infoList) +} + +func strGsubTable(L *LState, str string, repl *LTable, matches []*pm.MatchData) string { + infoList := make([]replaceInfo, 0, len(matches)) + for _, match := range matches { + idx := 0 + if match.CaptureLength() > 2 { // has captures + idx = 2 + } + var value LValue + if match.IsPosCapture(idx) { + value = L.GetTable(repl, LNumber(match.Capture(idx))) + } else { + value = L.GetField(repl, str[match.Capture(idx):match.Capture(idx+1)]) + } + if !LVIsFalse(value) { + infoList = append(infoList, replaceInfo{[]int{match.Capture(0), match.Capture(1)}, LVAsString(value)}) + } + } + return strGsubDoReplace(str, infoList) +} + +func strGsubFunc(L *LState, str string, repl *LFunction, matches []*pm.MatchData) string { + infoList := make([]replaceInfo, 0, len(matches)) + for _, match := range matches { + start, end := match.Capture(0), match.Capture(1) + L.Push(repl) + nargs := 0 + if match.CaptureLength() > 2 { // has captures + for i := 2; i < match.CaptureLength(); i += 2 { + if match.IsPosCapture(i) { + L.Push(LNumber(match.Capture(i))) + } else { + L.Push(LString(capturedString(L, match, str, i))) + } + nargs++ + } + } else { + L.Push(LString(capturedString(L, match, str, 0))) + nargs++ + } + L.Call(nargs, 1) + ret := L.reg.Pop() + if !LVIsFalse(ret) { + infoList = append(infoList, replaceInfo{[]int{start, end}, LVAsString(ret)}) + } + } + return strGsubDoReplace(str, infoList) +} + +type strMatchData struct { + str string + pos int + matches []*pm.MatchData +} + +func strGmatchIter(L *LState) int { + md := L.CheckUserData(1).Value.(*strMatchData) + str := md.str + matches := md.matches + idx := md.pos + md.pos += 1 + if idx == len(matches) { + return 0 + } + L.Push(L.Get(1)) + match := matches[idx] + if match.CaptureLength() == 2 { + L.Push(LString(str[match.Capture(0):match.Capture(1)])) + return 1 + } + + for i := 2; i < match.CaptureLength(); i += 2 { + if match.IsPosCapture(i) { + L.Push(LNumber(match.Capture(i))) + } else { + L.Push(LString(str[match.Capture(i):match.Capture(i+1)])) + } + } + return match.CaptureLength()/2 - 1 +} + +func strGmatch(L *LState) int { + str := L.CheckString(1) + pattern := L.CheckString(2) + mds, err := pm.Find(pattern, []byte(str), 0, -1) + if err != nil { + L.RaiseError(err.Error()) + } + L.Push(L.Get(UpvalueIndex(1))) + ud := L.NewUserData() + ud.Value = &strMatchData{str, 0, mds} + L.Push(ud) + return 2 +} + +func strLen(L *LState) int { + str := L.CheckString(1) + L.Push(LNumber(len(str))) + return 1 +} + +func strLower(L *LState) int { + str := L.CheckString(1) + L.Push(LString(strings.ToLower(str))) + return 1 +} + +func strMatch(L *LState) int { + str := L.CheckString(1) + pattern := L.CheckString(2) + offset := L.OptInt(3, 1) + l := len(str) + if offset < 0 { + offset = l + offset + 1 + } + offset-- + if offset < 0 { + offset = 0 + } + + mds, err := pm.Find(pattern, unsafeFastStringToReadOnlyBytes(str), offset, 1) + if err != nil { + L.RaiseError(err.Error()) + } + if len(mds) == 0 { + L.Push(LNil) + return 0 + } + md := mds[0] + nsubs := md.CaptureLength() / 2 + switch nsubs { + case 1: + L.Push(LString(str[md.Capture(0):md.Capture(1)])) + return 1 + default: + for i := 2; i < md.CaptureLength(); i += 2 { + if md.IsPosCapture(i) { + L.Push(LNumber(md.Capture(i))) + } else { + L.Push(LString(str[md.Capture(i):md.Capture(i+1)])) + } + } + return nsubs - 1 + } +} + +func strRep(L *LState) int { + str := L.CheckString(1) + n := L.CheckInt(2) + if n < 0 { + L.Push(emptyLString) + } else { + L.Push(LString(strings.Repeat(str, n))) + } + return 1 +} + +func strReverse(L *LState) int { + str := L.CheckString(1) + bts := []byte(str) + out := make([]byte, len(bts)) + for i, j := 0, len(bts)-1; j >= 0; i, j = i+1, j-1 { + out[i] = bts[j] + } + L.Push(LString(string(out))) + return 1 +} + +func strSub(L *LState) int { + str := L.CheckString(1) + start := luaIndex2StringIndex(str, L.CheckInt(2), true) + end := luaIndex2StringIndex(str, L.OptInt(3, -1), false) + l := len(str) + if start >= l || end < start { + L.Push(emptyLString) + } else { + L.Push(LString(str[start:end])) + } + return 1 +} + +func strUpper(L *LState) int { + str := L.CheckString(1) + L.Push(LString(strings.ToUpper(str))) + return 1 +} + +func luaIndex2StringIndex(str string, i int, start bool) int { + if start && i != 0 { + i -= 1 + } + l := len(str) + if i < 0 { + i = l + i + 1 + } + i = intMax(0, i) + if !start && i > l { + i = l + } + return i +} + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/table.go b/scraper-go/vendor/github.com/yuin/gopher-lua/table.go new file mode 100644 index 00000000..ddf14dd8 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/table.go @@ -0,0 +1,387 @@ +package lua + +const defaultArrayCap = 32 +const defaultHashCap = 32 + +type lValueArraySorter struct { + L *LState + Fn *LFunction + Values []LValue +} + +func (lv lValueArraySorter) Len() int { + return len(lv.Values) +} + +func (lv lValueArraySorter) Swap(i, j int) { + lv.Values[i], lv.Values[j] = lv.Values[j], lv.Values[i] +} + +func (lv lValueArraySorter) Less(i, j int) bool { + if lv.Fn != nil { + lv.L.Push(lv.Fn) + lv.L.Push(lv.Values[i]) + lv.L.Push(lv.Values[j]) + lv.L.Call(2, 1) + return LVAsBool(lv.L.reg.Pop()) + } + return lessThan(lv.L, lv.Values[i], lv.Values[j]) +} + +func newLTable(acap int, hcap int) *LTable { + if acap < 0 { + acap = 0 + } + if hcap < 0 { + hcap = 0 + } + tb := <able{} + tb.Metatable = LNil + if acap != 0 { + tb.array = make([]LValue, 0, acap) + } + if hcap != 0 { + tb.strdict = make(map[string]LValue, hcap) + } + return tb +} + +// Len returns length of this LTable without using __len. +func (tb *LTable) Len() int { + if tb.array == nil { + return 0 + } + var prev LValue = LNil + for i := len(tb.array) - 1; i >= 0; i-- { + v := tb.array[i] + if prev == LNil && v != LNil { + return i + 1 + } + prev = v + } + return 0 +} + +// Append appends a given LValue to this LTable. +func (tb *LTable) Append(value LValue) { + if value == LNil { + return + } + if tb.array == nil { + tb.array = make([]LValue, 0, defaultArrayCap) + } + if len(tb.array) == 0 || tb.array[len(tb.array)-1] != LNil { + tb.array = append(tb.array, value) + } else { + i := len(tb.array) - 2 + for ; i >= 0; i-- { + if tb.array[i] != LNil { + break + } + } + tb.array[i+1] = value + } +} + +// Insert inserts a given LValue at position `i` in this table. +func (tb *LTable) Insert(i int, value LValue) { + if tb.array == nil { + tb.array = make([]LValue, 0, defaultArrayCap) + } + if i > len(tb.array) { + tb.RawSetInt(i, value) + return + } + if i <= 0 { + tb.RawSet(LNumber(i), value) + return + } + i -= 1 + tb.array = append(tb.array, LNil) + copy(tb.array[i+1:], tb.array[i:]) + tb.array[i] = value +} + +// MaxN returns a maximum number key that nil value does not exist before it. +func (tb *LTable) MaxN() int { + if tb.array == nil { + return 0 + } + for i := len(tb.array) - 1; i >= 0; i-- { + if tb.array[i] != LNil { + return i + 1 + } + } + return 0 +} + +// Remove removes from this table the element at a given position. +func (tb *LTable) Remove(pos int) LValue { + if tb.array == nil { + return LNil + } + larray := len(tb.array) + if larray == 0 { + return LNil + } + i := pos - 1 + oldval := LNil + switch { + case i >= larray: + // nothing to do + case i == larray-1 || i < 0: + oldval = tb.array[larray-1] + tb.array = tb.array[:larray-1] + default: + oldval = tb.array[i] + copy(tb.array[i:], tb.array[i+1:]) + tb.array[larray-1] = nil + tb.array = tb.array[:larray-1] + } + return oldval +} + +// RawSet sets a given LValue to a given index without the __newindex metamethod. +// It is recommended to use `RawSetString` or `RawSetInt` for performance +// if you already know the given LValue is a string or number. +func (tb *LTable) RawSet(key LValue, value LValue) { + switch v := key.(type) { + case LNumber: + if isArrayKey(v) { + if tb.array == nil { + tb.array = make([]LValue, 0, defaultArrayCap) + } + index := int(v) - 1 + alen := len(tb.array) + switch { + case index == alen: + tb.array = append(tb.array, value) + case index > alen: + for i := 0; i < (index - alen); i++ { + tb.array = append(tb.array, LNil) + } + tb.array = append(tb.array, value) + case index < alen: + tb.array[index] = value + } + return + } + case LString: + tb.RawSetString(string(v), value) + return + } + + tb.RawSetH(key, value) +} + +// RawSetInt sets a given LValue at a position `key` without the __newindex metamethod. +func (tb *LTable) RawSetInt(key int, value LValue) { + if key < 1 || key >= MaxArrayIndex { + tb.RawSetH(LNumber(key), value) + return + } + if tb.array == nil { + tb.array = make([]LValue, 0, 32) + } + index := key - 1 + alen := len(tb.array) + switch { + case index == alen: + tb.array = append(tb.array, value) + case index > alen: + for i := 0; i < (index - alen); i++ { + tb.array = append(tb.array, LNil) + } + tb.array = append(tb.array, value) + case index < alen: + tb.array[index] = value + } +} + +// RawSetString sets a given LValue to a given string index without the __newindex metamethod. +func (tb *LTable) RawSetString(key string, value LValue) { + if tb.strdict == nil { + tb.strdict = make(map[string]LValue, defaultHashCap) + } + if tb.keys == nil { + tb.keys = []LValue{} + tb.k2i = map[LValue]int{} + } + + if value == LNil { + // TODO tb.keys and tb.k2i should also be removed + delete(tb.strdict, key) + } else { + tb.strdict[key] = value + lkey := LString(key) + if _, ok := tb.k2i[lkey]; !ok { + tb.k2i[lkey] = len(tb.keys) + tb.keys = append(tb.keys, lkey) + } + } +} + +// RawSetH sets a given LValue to a given index without the __newindex metamethod. +func (tb *LTable) RawSetH(key LValue, value LValue) { + if s, ok := key.(LString); ok { + tb.RawSetString(string(s), value) + return + } + if tb.dict == nil { + tb.dict = make(map[LValue]LValue, len(tb.strdict)) + } + if tb.keys == nil { + tb.keys = []LValue{} + tb.k2i = map[LValue]int{} + } + + if value == LNil { + // TODO tb.keys and tb.k2i should also be removed + delete(tb.dict, key) + } else { + tb.dict[key] = value + if _, ok := tb.k2i[key]; !ok { + tb.k2i[key] = len(tb.keys) + tb.keys = append(tb.keys, key) + } + } +} + +// RawGet returns an LValue associated with a given key without __index metamethod. +func (tb *LTable) RawGet(key LValue) LValue { + switch v := key.(type) { + case LNumber: + if isArrayKey(v) { + if tb.array == nil { + return LNil + } + index := int(v) - 1 + if index >= len(tb.array) { + return LNil + } + return tb.array[index] + } + case LString: + if tb.strdict == nil { + return LNil + } + if ret, ok := tb.strdict[string(v)]; ok { + return ret + } + return LNil + } + if tb.dict == nil { + return LNil + } + if v, ok := tb.dict[key]; ok { + return v + } + return LNil +} + +// RawGetInt returns an LValue at position `key` without __index metamethod. +func (tb *LTable) RawGetInt(key int) LValue { + if tb.array == nil { + return LNil + } + index := int(key) - 1 + if index >= len(tb.array) || index < 0 { + return LNil + } + return tb.array[index] +} + +// RawGet returns an LValue associated with a given key without __index metamethod. +func (tb *LTable) RawGetH(key LValue) LValue { + if s, sok := key.(LString); sok { + if tb.strdict == nil { + return LNil + } + if v, vok := tb.strdict[string(s)]; vok { + return v + } + return LNil + } + if tb.dict == nil { + return LNil + } + if v, ok := tb.dict[key]; ok { + return v + } + return LNil +} + +// RawGetString returns an LValue associated with a given key without __index metamethod. +func (tb *LTable) RawGetString(key string) LValue { + if tb.strdict == nil { + return LNil + } + if v, vok := tb.strdict[string(key)]; vok { + return v + } + return LNil +} + +// ForEach iterates over this table of elements, yielding each in turn to a given function. +func (tb *LTable) ForEach(cb func(LValue, LValue)) { + if tb.array != nil { + for i, v := range tb.array { + if v != LNil { + cb(LNumber(i+1), v) + } + } + } + if tb.strdict != nil { + for k, v := range tb.strdict { + if v != LNil { + cb(LString(k), v) + } + } + } + if tb.dict != nil { + for k, v := range tb.dict { + if v != LNil { + cb(k, v) + } + } + } +} + +// This function is equivalent to lua_next ( http://www.lua.org/manual/5.1/manual.html#lua_next ). +func (tb *LTable) Next(key LValue) (LValue, LValue) { + init := false + if key == LNil { + key = LNumber(0) + init = true + } + + if init || key != LNumber(0) { + if kv, ok := key.(LNumber); ok && isInteger(kv) && int(kv) >= 0 && kv < LNumber(MaxArrayIndex) { + index := int(kv) + if tb.array != nil { + for ; index < len(tb.array); index++ { + if v := tb.array[index]; v != LNil { + return LNumber(index + 1), v + } + } + } + if tb.array == nil || index == len(tb.array) { + if (tb.dict == nil || len(tb.dict) == 0) && (tb.strdict == nil || len(tb.strdict) == 0) { + return LNil, LNil + } + key = tb.keys[0] + if v := tb.RawGetH(key); v != LNil { + return key, v + } + } + } + } + + for i := tb.k2i[key] + 1; i < len(tb.keys); i++ { + key := tb.keys[i] + if v := tb.RawGetH(key); v != LNil { + return key, v + } + } + return LNil, LNil +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/tablelib.go b/scraper-go/vendor/github.com/yuin/gopher-lua/tablelib.go new file mode 100644 index 00000000..f3f46070 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/tablelib.go @@ -0,0 +1,100 @@ +package lua + +import ( + "sort" +) + +func OpenTable(L *LState) int { + tabmod := L.RegisterModule(TabLibName, tableFuncs) + L.Push(tabmod) + return 1 +} + +var tableFuncs = map[string]LGFunction{ + "getn": tableGetN, + "concat": tableConcat, + "insert": tableInsert, + "maxn": tableMaxN, + "remove": tableRemove, + "sort": tableSort, +} + +func tableSort(L *LState) int { + tbl := L.CheckTable(1) + sorter := lValueArraySorter{L, nil, tbl.array} + if L.GetTop() != 1 { + sorter.Fn = L.CheckFunction(2) + } + sort.Sort(sorter) + return 0 +} + +func tableGetN(L *LState) int { + L.Push(LNumber(L.CheckTable(1).Len())) + return 1 +} + +func tableMaxN(L *LState) int { + L.Push(LNumber(L.CheckTable(1).MaxN())) + return 1 +} + +func tableRemove(L *LState) int { + tbl := L.CheckTable(1) + if L.GetTop() == 1 { + L.Push(tbl.Remove(-1)) + } else { + L.Push(tbl.Remove(L.CheckInt(2))) + } + return 1 +} + +func tableConcat(L *LState) int { + tbl := L.CheckTable(1) + sep := LString(L.OptString(2, "")) + i := L.OptInt(3, 1) + j := L.OptInt(4, tbl.Len()) + if L.GetTop() == 3 { + if i > tbl.Len() || i < 1 { + L.Push(emptyLString) + return 1 + } + } + i = intMax(intMin(i, tbl.Len()), 1) + j = intMin(intMin(j, tbl.Len()), tbl.Len()) + if i > j { + L.Push(emptyLString) + return 1 + } + //TODO should flushing? + retbottom := L.GetTop() + for ; i <= j; i++ { + v := tbl.RawGetInt(i) + if !LVCanConvToString(v) { + L.RaiseError("invalid value (%s) at index %d in table for concat", v.Type().String(), i) + } + L.Push(v) + if i != j { + L.Push(sep) + } + } + L.Push(stringConcat(L, L.GetTop()-retbottom, L.reg.Top()-1)) + return 1 +} + +func tableInsert(L *LState) int { + tbl := L.CheckTable(1) + nargs := L.GetTop() + if nargs == 1 { + L.RaiseError("wrong number of arguments") + } + + if L.GetTop() == 2 { + tbl.Append(L.Get(2)) + return 0 + } + tbl.Insert(int(L.CheckInt(2)), L.CheckAny(3)) + return 0 +} + +// diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/utils.go b/scraper-go/vendor/github.com/yuin/gopher-lua/utils.go new file mode 100644 index 00000000..2df68dc7 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/utils.go @@ -0,0 +1,265 @@ +package lua + +import ( + "bufio" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "time" + "unsafe" +) + +func intMin(a, b int) int { + if a < b { + return a + } else { + return b + } +} + +func intMax(a, b int) int { + if a > b { + return a + } else { + return b + } +} + +func defaultFormat(v interface{}, f fmt.State, c rune) { + buf := make([]string, 0, 10) + buf = append(buf, "%") + for i := 0; i < 128; i++ { + if f.Flag(i) { + buf = append(buf, string(rune(i))) + } + } + + if w, ok := f.Width(); ok { + buf = append(buf, strconv.Itoa(w)) + } + if p, ok := f.Precision(); ok { + buf = append(buf, "."+strconv.Itoa(p)) + } + buf = append(buf, string(c)) + format := strings.Join(buf, "") + fmt.Fprintf(f, format, v) +} + +type flagScanner struct { + flag byte + start string + end string + buf []byte + str string + Length int + Pos int + HasFlag bool + ChangeFlag bool +} + +func newFlagScanner(flag byte, start, end, str string) *flagScanner { + return &flagScanner{flag, start, end, make([]byte, 0, len(str)), str, len(str), 0, false, false} +} + +func (fs *flagScanner) AppendString(str string) { fs.buf = append(fs.buf, str...) } + +func (fs *flagScanner) AppendChar(ch byte) { fs.buf = append(fs.buf, ch) } + +func (fs *flagScanner) String() string { return string(fs.buf) } + +func (fs *flagScanner) Next() (byte, bool) { + c := byte('\000') + fs.ChangeFlag = false + if fs.Pos == fs.Length { + if fs.HasFlag { + fs.AppendString(fs.end) + } + return c, true + } else { + c = fs.str[fs.Pos] + if c == fs.flag { + if fs.Pos < (fs.Length-1) && fs.str[fs.Pos+1] == fs.flag { + fs.HasFlag = false + fs.AppendChar(fs.flag) + fs.Pos += 2 + return fs.Next() + } else if fs.Pos != fs.Length-1 { + if fs.HasFlag { + fs.AppendString(fs.end) + } + fs.AppendString(fs.start) + fs.ChangeFlag = true + fs.HasFlag = true + } + } + } + fs.Pos++ + return c, false +} + +var cDateFlagToGo = map[byte]string{ + 'a': "mon", 'A': "Monday", 'b': "Jan", 'B': "January", 'c': "02 Jan 06 15:04 MST", 'd': "02", + 'F': "2006-01-02", 'H': "15", 'I': "03", 'm': "01", 'M': "04", 'p': "PM", 'P': "pm", 'S': "05", + 'x': "15/04/05", 'X': "15:04:05", 'y': "06", 'Y': "2006", 'z': "-0700", 'Z': "MST"} + +func strftime(t time.Time, cfmt string) string { + sc := newFlagScanner('%', "", "", cfmt) + for c, eos := sc.Next(); !eos; c, eos = sc.Next() { + if !sc.ChangeFlag { + if sc.HasFlag { + if v, ok := cDateFlagToGo[c]; ok { + sc.AppendString(t.Format(v)) + } else { + switch c { + case 'w': + sc.AppendString(fmt.Sprint(int(t.Weekday()))) + default: + sc.AppendChar('%') + sc.AppendChar(c) + } + } + sc.HasFlag = false + } else { + sc.AppendChar(c) + } + } + } + + return sc.String() +} + +func isInteger(v LNumber) bool { + return float64(v) == float64(int64(v)) + //_, frac := math.Modf(float64(v)) + //return frac == 0.0 +} + +func isArrayKey(v LNumber) bool { + return isInteger(v) && v < LNumber(int((^uint(0))>>1)) && v > LNumber(0) && v < LNumber(MaxArrayIndex) +} + +func parseNumber(number string) (LNumber, error) { + var value LNumber + number = strings.Trim(number, " \t\n") + if v, err := strconv.ParseInt(number, 0, LNumberBit); err != nil { + if v2, err2 := strconv.ParseFloat(number, LNumberBit); err2 != nil { + return LNumber(0), err2 + } else { + value = LNumber(v2) + } + } else { + value = LNumber(v) + } + return value, nil +} + +func popenArgs(arg string) (string, []string) { + cmd := "/bin/sh" + args := []string{"-c"} + if LuaOS == "windows" { + cmd = "C:\\Windows\\system32\\cmd.exe" + args = []string{"/c"} + } + args = append(args, arg) + return cmd, args +} + +func isGoroutineSafe(lv LValue) bool { + switch v := lv.(type) { + case *LFunction, *LUserData, *LState: + return false + case *LTable: + return v.Metatable == LNil + default: + return true + } +} + +func readBufioSize(reader *bufio.Reader, size int64) ([]byte, error, bool) { + result := []byte{} + read := int64(0) + var err error + var n int + for read != size { + buf := make([]byte, size-read) + n, err = reader.Read(buf) + if err != nil { + break + } + read += int64(n) + result = append(result, buf[:n]...) + } + e := err + if e != nil && e == io.EOF { + e = nil + } + + return result, e, len(result) == 0 && err == io.EOF +} + +func readBufioLine(reader *bufio.Reader) ([]byte, error, bool) { + result := []byte{} + var buf []byte + var err error + var isprefix bool = true + for isprefix { + buf, isprefix, err = reader.ReadLine() + if err != nil { + break + } + result = append(result, buf...) + } + e := err + if e != nil && e == io.EOF { + e = nil + } + + return result, e, len(result) == 0 && err == io.EOF +} + +func int2Fb(val int) int { + e := 0 + x := val + for x >= 16 { + x = (x + 1) >> 1 + e++ + } + if x < 8 { + return x + } + return ((e + 1) << 3) | (x - 8) +} + +func strCmp(s1, s2 string) int { + len1 := len(s1) + len2 := len(s2) + for i := 0; ; i++ { + c1 := -1 + if i < len1 { + c1 = int(s1[i]) + } + c2 := -1 + if i != len2 { + c2 = int(s2[i]) + } + switch { + case c1 < c2: + return -1 + case c1 > c2: + return +1 + case c1 < 0: + return 0 + } + } +} + +func unsafeFastStringToReadOnlyBytes(s string) (bs []byte) { + sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) + bh := (*reflect.SliceHeader)(unsafe.Pointer(&bs)) + bh.Data = sh.Data + bh.Cap = sh.Len + bh.Len = sh.Len + return +} diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/value.go b/scraper-go/vendor/github.com/yuin/gopher-lua/value.go new file mode 100644 index 00000000..4156e9d5 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/value.go @@ -0,0 +1,215 @@ +package lua + +import ( + "context" + "fmt" + "os" +) + +type LValueType int + +const ( + LTNil LValueType = iota + LTBool + LTNumber + LTString + LTFunction + LTUserData + LTThread + LTTable + LTChannel +) + +var lValueNames = [9]string{"nil", "boolean", "number", "string", "function", "userdata", "thread", "table", "channel"} + +func (vt LValueType) String() string { + return lValueNames[int(vt)] +} + +type LValue interface { + String() string + Type() LValueType +} + +// LVIsFalse returns true if a given LValue is a nil or false otherwise false. +func LVIsFalse(v LValue) bool { return v == LNil || v == LFalse } + +// LVIsFalse returns false if a given LValue is a nil or false otherwise true. +func LVAsBool(v LValue) bool { return v != LNil && v != LFalse } + +// LVAsString returns string representation of a given LValue +// if the LValue is a string or number, otherwise an empty string. +func LVAsString(v LValue) string { + switch sn := v.(type) { + case LString, LNumber: + return sn.String() + default: + return "" + } +} + +// LVCanConvToString returns true if a given LValue is a string or number +// otherwise false. +func LVCanConvToString(v LValue) bool { + switch v.(type) { + case LString, LNumber: + return true + default: + return false + } +} + +// LVAsNumber tries to convert a given LValue to a number. +func LVAsNumber(v LValue) LNumber { + switch lv := v.(type) { + case LNumber: + return lv + case LString: + if num, err := parseNumber(string(lv)); err == nil { + return num + } + } + return LNumber(0) +} + +type LNilType struct{} + +func (nl *LNilType) String() string { return "nil" } +func (nl *LNilType) Type() LValueType { return LTNil } + +var LNil = LValue(&LNilType{}) + +type LBool bool + +func (bl LBool) String() string { + if bool(bl) { + return "true" + } + return "false" +} +func (bl LBool) Type() LValueType { return LTBool } + +var LTrue = LBool(true) +var LFalse = LBool(false) + +type LString string + +func (st LString) String() string { return string(st) } +func (st LString) Type() LValueType { return LTString } + +// fmt.Formatter interface +func (st LString) Format(f fmt.State, c rune) { + switch c { + case 'd', 'i': + if nm, err := parseNumber(string(st)); err != nil { + defaultFormat(nm, f, 'd') + } else { + defaultFormat(string(st), f, 's') + } + default: + defaultFormat(string(st), f, c) + } +} + +func (nm LNumber) String() string { + if isInteger(nm) { + return fmt.Sprint(int64(nm)) + } + return fmt.Sprint(float64(nm)) +} + +func (nm LNumber) Type() LValueType { return LTNumber } + +// fmt.Formatter interface +func (nm LNumber) Format(f fmt.State, c rune) { + switch c { + case 'q', 's': + defaultFormat(nm.String(), f, c) + case 'b', 'c', 'd', 'o', 'x', 'X', 'U': + defaultFormat(int64(nm), f, c) + case 'e', 'E', 'f', 'F', 'g', 'G': + defaultFormat(float64(nm), f, c) + case 'i': + defaultFormat(int64(nm), f, 'd') + default: + if isInteger(nm) { + defaultFormat(int64(nm), f, c) + } else { + defaultFormat(float64(nm), f, c) + } + } +} + +type LTable struct { + Metatable LValue + + array []LValue + dict map[LValue]LValue + strdict map[string]LValue + keys []LValue + k2i map[LValue]int +} + +func (tb *LTable) String() string { return fmt.Sprintf("table: %p", tb) } +func (tb *LTable) Type() LValueType { return LTTable } + +type LFunction struct { + IsG bool + Env *LTable + Proto *FunctionProto + GFunction LGFunction + Upvalues []*Upvalue +} +type LGFunction func(*LState) int + +func (fn *LFunction) String() string { return fmt.Sprintf("function: %p", fn) } +func (fn *LFunction) Type() LValueType { return LTFunction } + +type Global struct { + MainThread *LState + CurrentThread *LState + Registry *LTable + Global *LTable + + builtinMts map[int]LValue + tempFiles []*os.File + gccount int32 +} + +type LState struct { + G *Global + Parent *LState + Env *LTable + Panic func(*LState) + Dead bool + Options Options + + stop int32 + reg *registry + stack callFrameStack + alloc *allocator + currentFrame *callFrame + wrapped bool + uvcache *Upvalue + hasErrorFunc bool + mainLoop func(*LState, *callFrame) + ctx context.Context + ctxCancelFn context.CancelFunc +} + +func (ls *LState) String() string { return fmt.Sprintf("thread: %p", ls) } +func (ls *LState) Type() LValueType { return LTThread } + +type LUserData struct { + Value interface{} + Env *LTable + Metatable LValue +} + +func (ud *LUserData) String() string { return fmt.Sprintf("userdata: %p", ud) } +func (ud *LUserData) Type() LValueType { return LTUserData } + +type LChannel chan LValue + +func (ch LChannel) String() string { return fmt.Sprintf("channel: %p", ch) } +func (ch LChannel) Type() LValueType { return LTChannel } diff --git a/scraper-go/vendor/github.com/yuin/gopher-lua/vm.go b/scraper-go/vendor/github.com/yuin/gopher-lua/vm.go new file mode 100644 index 00000000..97335a75 --- /dev/null +++ b/scraper-go/vendor/github.com/yuin/gopher-lua/vm.go @@ -0,0 +1,2465 @@ +package lua + +//////////////////////////////////////////////////////// +// This file was generated by go-inline. DO NOT EDIT. // +//////////////////////////////////////////////////////// + +import ( + "fmt" + "math" + "strings" +) + +func mainLoop(L *LState, baseframe *callFrame) { + var inst uint32 + var cf *callFrame + + if L.stack.IsEmpty() { + return + } + + L.currentFrame = L.stack.Last() + if L.currentFrame.Fn.IsG { + callGFunction(L, false) + return + } + + for { + cf = L.currentFrame + inst = cf.Fn.Proto.Code[cf.Pc] + cf.Pc++ + if jumpTable[int(inst>>26)](L, inst, baseframe) == 1 { + return + } + } +} + +func mainLoopWithContext(L *LState, baseframe *callFrame) { + var inst uint32 + var cf *callFrame + + if L.stack.IsEmpty() { + return + } + + L.currentFrame = L.stack.Last() + if L.currentFrame.Fn.IsG { + callGFunction(L, false) + return + } + + for { + cf = L.currentFrame + inst = cf.Fn.Proto.Code[cf.Pc] + cf.Pc++ + select { + case <-L.ctx.Done(): + L.RaiseError(L.ctx.Err().Error()) + return + default: + if jumpTable[int(inst>>26)](L, inst, baseframe) == 1 { + return + } + } + } +} + +// regv is the first target register to copy the return values to. +// It can be reg.top, indicating that the copied values are going into new registers, or it can be below reg.top +// Indicating that the values should be within the existing registers. +// b is the available number of return values + 1. +// n is the desired number of return values. +// If n more than the available return values then the extra values are set to nil. +// When this function returns the top of the registry will be set to regv+n. +func copyReturnValues(L *LState, regv, start, n, b int) { // +inline-start + if b == 1 { + // this section is inlined by go-inline + // source function is 'func (rg *registry) FillNil(regm, n int) ' in '_state.go' + { + rg := L.reg + regm := regv + newSize := regm + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := 0; i < n; i++ { + rg.array[regm+i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regm + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + } else { + // this section is inlined by go-inline + // source function is 'func (rg *registry) CopyRange(regv, start, limit, n int) ' in '_state.go' + { + rg := L.reg + limit := -1 + newSize := regv + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + if limit == -1 || limit > rg.top { + limit = rg.top + } + for i := 0; i < n; i++ { + srcIdx := start + i + if srcIdx >= limit || srcIdx < 0 { + rg.array[regv+i] = LNil + } else { + rg.array[regv+i] = rg.array[srcIdx] + } + } + + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regv + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + if b > 1 && n > (b-1) { + // this section is inlined by go-inline + // source function is 'func (rg *registry) FillNil(regm, n int) ' in '_state.go' + { + rg := L.reg + regm := regv + b - 1 + n := n - (b - 1) + newSize := regm + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := 0; i < n; i++ { + rg.array[regm+i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regm + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + } + } +} // +inline-end + +func switchToParentThread(L *LState, nargs int, haserror bool, kill bool) { + parent := L.Parent + if parent == nil { + L.RaiseError("can not yield from outside of a coroutine") + } + L.G.CurrentThread = parent + L.Parent = nil + if !L.wrapped { + if haserror { + parent.Push(LFalse) + } else { + parent.Push(LTrue) + } + } + L.XMoveTo(parent, nargs) + L.stack.Pop() + offset := L.currentFrame.LocalBase - L.currentFrame.ReturnBase + L.currentFrame = L.stack.Last() + L.reg.SetTop(L.reg.Top() - offset) // remove 'yield' function(including tailcalled functions) + if kill { + L.kill() + } +} + +func callGFunction(L *LState, tailcall bool) bool { + frame := L.currentFrame + gfnret := frame.Fn.GFunction(L) + if tailcall { + L.currentFrame = L.RemoveCallerFrame() + } + + if gfnret < 0 { + switchToParentThread(L, L.GetTop(), false, false) + return true + } + + wantret := frame.NRet + if wantret == MultRet { + wantret = gfnret + } + + if tailcall && L.Parent != nil && L.stack.Sp() == 1 { + switchToParentThread(L, wantret, false, true) + return true + } + + // this section is inlined by go-inline + // source function is 'func (rg *registry) CopyRange(regv, start, limit, n int) ' in '_state.go' + { + rg := L.reg + regv := frame.ReturnBase + start := L.reg.Top() - gfnret + limit := -1 + n := wantret + newSize := regv + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + if limit == -1 || limit > rg.top { + limit = rg.top + } + for i := 0; i < n; i++ { + srcIdx := start + i + if srcIdx >= limit || srcIdx < 0 { + rg.array[regv+i] = LNil + } else { + rg.array[regv+i] = rg.array[srcIdx] + } + } + + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regv + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + L.stack.Pop() + L.currentFrame = L.stack.Last() + return false +} + +func threadRun(L *LState) { + if L.stack.IsEmpty() { + return + } + + defer func() { + if rcv := recover(); rcv != nil { + var lv LValue + if v, ok := rcv.(*ApiError); ok { + lv = v.Object + } else { + lv = LString(fmt.Sprint(rcv)) + } + if parent := L.Parent; parent != nil { + if L.wrapped { + L.Push(lv) + parent.Panic(L) + } else { + L.SetTop(0) + L.Push(lv) + switchToParentThread(L, 1, true, true) + } + } else { + panic(rcv) + } + } + }() + L.mainLoop(L, nil) +} + +type instFunc func(*LState, uint32, *callFrame) int + +var jumpTable [opCodeMax + 1]instFunc + +func init() { + jumpTable = [opCodeMax + 1]instFunc{ + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_MOVE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + v := reg.Get(lbase + B) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_MOVEN + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + v := reg.Get(lbase + B) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := lbase + A + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + code := cf.Fn.Proto.Code + pc := cf.Pc + for i := 0; i < C; i++ { + inst = code[pc] + pc++ + A = int(inst>>18) & 0xff //GETA + B = int(inst & 0x1ff) //GETB + v := reg.Get(lbase + B) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := lbase + A + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } + cf.Pc = pc + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LOADK + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Bx := int(inst & 0x3ffff) //GETBX + v := cf.Fn.Proto.Constants[Bx] + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LOADBOOL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + if B != 0 { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := LTrue + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := LFalse + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } + if C != 0 { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LOADNIL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + for i := RA; i <= lbase+B; i++ { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := i + vali := LNil + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_GETUPVAL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + v := cf.Fn.Upvalues[B].Value() + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_GETGLOBAL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Bx := int(inst & 0x3ffff) //GETBX + //reg.Set(RA, L.getField(cf.Fn.Env, cf.Fn.Proto.Constants[Bx])) + v := L.getFieldString(cf.Fn.Env, cf.Fn.Proto.stringConstants[Bx]) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_GETTABLE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + v := L.getField(reg.Get(lbase+B), L.rkValue(C)) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_GETTABLEKS + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + v := L.getFieldString(reg.Get(lbase+B), L.rkString(C)) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETGLOBAL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Bx := int(inst & 0x3ffff) //GETBX + //L.setField(cf.Fn.Env, cf.Fn.Proto.Constants[Bx], reg.Get(RA)) + L.setFieldString(cf.Fn.Env, cf.Fn.Proto.stringConstants[Bx], reg.Get(RA)) + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETUPVAL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + cf.Fn.Upvalues[B].SetValue(reg.Get(RA)) + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETTABLE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + L.setField(reg.Get(RA), L.rkValue(B), L.rkValue(C)) + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETTABLEKS + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + L.setFieldString(reg.Get(RA), L.rkString(B), L.rkValue(C)) + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_NEWTABLE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + v := newLTable(B, C) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SELF + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + selfobj := reg.Get(lbase + B) + v := L.getFieldString(selfobj, L.rkString(C)) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + 1 + vali := selfobj + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + opArith, // OP_ADD + opArith, // OP_SUB + opArith, // OP_MUL + opArith, // OP_DIV + opArith, // OP_MOD + opArith, // OP_POW + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_UNM + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + unaryv := L.rkValue(B) + if nm, ok := unaryv.(LNumber); ok { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := -nm + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + op := L.metaOp1(unaryv, "__unm") + if op.Type() == LTFunction { + reg.Push(op) + reg.Push(unaryv) + L.Call(1, 1) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := reg.Pop() + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else if str, ok1 := unaryv.(LString); ok1 { + if num, err := parseNumber(string(str)); err == nil { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := -num + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + L.RaiseError("__unm undefined") + } + } else { + L.RaiseError("__unm undefined") + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_NOT + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + if LVIsFalse(reg.Get(lbase + B)) { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := LTrue + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := LFalse + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LEN + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + switch lv := L.rkValue(B).(type) { + case LString: + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetNumber(regi int, vali LNumber) ' in '_state.go' + { + rg := reg + regi := RA + vali := LNumber(len(lv)) + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } + } + default: + op := L.metaOp1(lv, "__len") + if op.Type() == LTFunction { + reg.Push(op) + reg.Push(lv) + L.Call(1, 1) + ret := reg.Pop() + if ret.Type() == LTNumber { + v, _ := ret.(LNumber) + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetNumber(regi int, vali LNumber) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := ret + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } + } else if lv.Type() == LTTable { + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetNumber(regi int, vali LNumber) ' in '_state.go' + { + rg := reg + regi := RA + vali := LNumber(lv.(*LTable).Len()) + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + L.RaiseError("__len undefined") + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_CONCAT + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + RC := lbase + C + RB := lbase + B + v := stringConcat(L, RC-RB+1, RC) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_JMP + cf := L.currentFrame + Sbx := int(inst&0x3ffff) - opMaxArgSbx //GETSBX + cf.Pc += Sbx + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_EQ + cf := L.currentFrame + A := int(inst>>18) & 0xff //GETA + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + ret := equals(L, L.rkValue(B), L.rkValue(C), false) + v := 1 + if ret { + v = 0 + } + if v == A { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LT + cf := L.currentFrame + A := int(inst>>18) & 0xff //GETA + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + ret := lessThan(L, L.rkValue(B), L.rkValue(C)) + v := 1 + if ret { + v = 0 + } + if v == A { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_LE + cf := L.currentFrame + A := int(inst>>18) & 0xff //GETA + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + lhs := L.rkValue(B) + rhs := L.rkValue(C) + ret := false + + if v1, ok1 := lhs.(LNumber); ok1 { + if v2, ok2 := rhs.(LNumber); ok2 { + ret = v1 <= v2 + } else { + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + } + } else { + if lhs.Type() != rhs.Type() { + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + } + switch lhs.Type() { + case LTString: + ret = strCmp(string(lhs.(LString)), string(rhs.(LString))) <= 0 + default: + switch objectRational(L, lhs, rhs, "__le") { + case 1: + ret = true + case 0: + ret = false + default: + ret = !objectRationalWithError(L, rhs, lhs, "__lt") + } + } + } + + v := 1 + if ret { + v = 0 + } + if v == A { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_TEST + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + C := int(inst>>9) & 0x1ff //GETC + if LVAsBool(reg.Get(RA)) == (C == 0) { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_TESTSET + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + if value := reg.Get(lbase + B); LVAsBool(value) != (C == 0) { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := value + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + cf.Pc++ + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_CALL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + nargs := B - 1 + if B == 0 { + nargs = reg.Top() - (RA + 1) + } + lv := reg.Get(RA) + nret := C - 1 + var callable *LFunction + var meta bool + if fn, ok := lv.(*LFunction); ok { + callable = fn + meta = false + } else { + callable, meta = L.metaCall(lv) + } + // this section is inlined by go-inline + // source function is 'func (ls *LState) pushCallFrame(cf callFrame, fn LValue, meta bool) ' in '_state.go' + { + ls := L + cf := callFrame{Fn: callable, Pc: 0, Base: RA, LocalBase: RA + 1, ReturnBase: RA, NArgs: nargs, NRet: nret, Parent: cf, TailCall: 0} + fn := lv + if meta { + cf.NArgs++ + ls.reg.Insert(fn, cf.LocalBase) + } + if cf.Fn == nil { + ls.RaiseError("attempt to call a non-function object") + } + if ls.stack.IsFull() { + ls.RaiseError("stack overflow") + } + ls.stack.Push(cf) + newcf := ls.stack.Last() + // this section is inlined by go-inline + // source function is 'func (ls *LState) initCallFrame(cf *callFrame) ' in '_state.go' + { + cf := newcf + if cf.Fn.IsG { + ls.reg.SetTop(cf.LocalBase + cf.NArgs) + } else { + proto := cf.Fn.Proto + nargs := cf.NArgs + np := int(proto.NumParameters) + if nargs < np { + // default any missing arguments to nil + newSize := cf.LocalBase + np + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + rg := ls.reg + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := nargs; i < np; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + nargs = np + ls.reg.top = newSize + } + + if (proto.IsVarArg & VarArgIsVarArg) == 0 { + if nargs < int(proto.NumUsedRegisters) { + nargs = int(proto.NumUsedRegisters) + } + newSize := cf.LocalBase + nargs + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + rg := ls.reg + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := np; i < nargs; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + ls.reg.top = cf.LocalBase + int(proto.NumUsedRegisters) + } else { + /* swap vararg positions: + closure + namedparam1 <- lbase + namedparam2 + vararg1 + vararg2 + + TO + + closure + nil + nil + vararg1 + vararg2 + namedparam1 <- lbase + namedparam2 + */ + nvarargs := nargs - np + if nvarargs < 0 { + nvarargs = 0 + } + + ls.reg.SetTop(cf.LocalBase + nargs + np) + for i := 0; i < np; i++ { + //ls.reg.Set(cf.LocalBase+nargs+i, ls.reg.Get(cf.LocalBase+i)) + ls.reg.array[cf.LocalBase+nargs+i] = ls.reg.array[cf.LocalBase+i] + //ls.reg.Set(cf.LocalBase+i, LNil) + ls.reg.array[cf.LocalBase+i] = LNil + } + + if CompatVarArg { + ls.reg.SetTop(cf.LocalBase + nargs + np + 1) + if (proto.IsVarArg & VarArgNeedsArg) != 0 { + argtb := newLTable(nvarargs, 0) + for i := 0; i < nvarargs; i++ { + argtb.RawSetInt(i+1, ls.reg.Get(cf.LocalBase+np+i)) + } + argtb.RawSetString("n", LNumber(nvarargs)) + //ls.reg.Set(cf.LocalBase+nargs+np, argtb) + ls.reg.array[cf.LocalBase+nargs+np] = argtb + } else { + ls.reg.array[cf.LocalBase+nargs+np] = LNil + } + } + cf.LocalBase += nargs + maxreg := cf.LocalBase + int(proto.NumUsedRegisters) + ls.reg.SetTop(maxreg) + } + } + } + ls.currentFrame = newcf + } + if callable.IsG && callGFunction(L, false) { + return 1 + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_TAILCALL + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + nargs := B - 1 + if B == 0 { + nargs = reg.Top() - (RA + 1) + } + lv := reg.Get(RA) + var callable *LFunction + var meta bool + if fn, ok := lv.(*LFunction); ok { + callable = fn + meta = false + } else { + callable, meta = L.metaCall(lv) + } + if callable == nil { + L.RaiseError("attempt to call a non-function object") + } + // this section is inlined by go-inline + // source function is 'func (ls *LState) closeUpvalues(idx int) ' in '_state.go' + { + ls := L + idx := lbase + if ls.uvcache != nil { + var prev *Upvalue + for uv := ls.uvcache; uv != nil; uv = uv.next { + if uv.index >= idx { + if prev != nil { + prev.next = nil + } else { + ls.uvcache = nil + } + uv.Close() + } + prev = uv + } + } + } + if callable.IsG { + luaframe := cf + L.pushCallFrame(callFrame{ + Fn: callable, + Pc: 0, + Base: RA, + LocalBase: RA + 1, + ReturnBase: cf.ReturnBase, + NArgs: nargs, + NRet: cf.NRet, + Parent: cf, + TailCall: 0, + }, lv, meta) + if callGFunction(L, true) { + return 1 + } + if L.currentFrame == nil || L.currentFrame.Fn.IsG || luaframe == baseframe { + return 1 + } + } else { + base := cf.Base + cf.Fn = callable + cf.Pc = 0 + cf.Base = RA + cf.LocalBase = RA + 1 + cf.ReturnBase = cf.ReturnBase + cf.NArgs = nargs + cf.NRet = cf.NRet + cf.TailCall++ + lbase := cf.LocalBase + if meta { + cf.NArgs++ + L.reg.Insert(lv, cf.LocalBase) + } + // this section is inlined by go-inline + // source function is 'func (ls *LState) initCallFrame(cf *callFrame) ' in '_state.go' + { + ls := L + if cf.Fn.IsG { + ls.reg.SetTop(cf.LocalBase + cf.NArgs) + } else { + proto := cf.Fn.Proto + nargs := cf.NArgs + np := int(proto.NumParameters) + if nargs < np { + // default any missing arguments to nil + newSize := cf.LocalBase + np + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + rg := ls.reg + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := nargs; i < np; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + nargs = np + ls.reg.top = newSize + } + + if (proto.IsVarArg & VarArgIsVarArg) == 0 { + if nargs < int(proto.NumUsedRegisters) { + nargs = int(proto.NumUsedRegisters) + } + newSize := cf.LocalBase + nargs + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + rg := ls.reg + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := np; i < nargs; i++ { + ls.reg.array[cf.LocalBase+i] = LNil + } + ls.reg.top = cf.LocalBase + int(proto.NumUsedRegisters) + } else { + /* swap vararg positions: + closure + namedparam1 <- lbase + namedparam2 + vararg1 + vararg2 + + TO + + closure + nil + nil + vararg1 + vararg2 + namedparam1 <- lbase + namedparam2 + */ + nvarargs := nargs - np + if nvarargs < 0 { + nvarargs = 0 + } + + ls.reg.SetTop(cf.LocalBase + nargs + np) + for i := 0; i < np; i++ { + //ls.reg.Set(cf.LocalBase+nargs+i, ls.reg.Get(cf.LocalBase+i)) + ls.reg.array[cf.LocalBase+nargs+i] = ls.reg.array[cf.LocalBase+i] + //ls.reg.Set(cf.LocalBase+i, LNil) + ls.reg.array[cf.LocalBase+i] = LNil + } + + if CompatVarArg { + ls.reg.SetTop(cf.LocalBase + nargs + np + 1) + if (proto.IsVarArg & VarArgNeedsArg) != 0 { + argtb := newLTable(nvarargs, 0) + for i := 0; i < nvarargs; i++ { + argtb.RawSetInt(i+1, ls.reg.Get(cf.LocalBase+np+i)) + } + argtb.RawSetString("n", LNumber(nvarargs)) + //ls.reg.Set(cf.LocalBase+nargs+np, argtb) + ls.reg.array[cf.LocalBase+nargs+np] = argtb + } else { + ls.reg.array[cf.LocalBase+nargs+np] = LNil + } + } + cf.LocalBase += nargs + maxreg := cf.LocalBase + int(proto.NumUsedRegisters) + ls.reg.SetTop(maxreg) + } + } + } + // this section is inlined by go-inline + // source function is 'func (rg *registry) CopyRange(regv, start, limit, n int) ' in '_state.go' + { + rg := L.reg + regv := base + start := RA + limit := -1 + n := reg.Top() - RA - 1 + newSize := regv + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + if limit == -1 || limit > rg.top { + limit = rg.top + } + for i := 0; i < n; i++ { + srcIdx := start + i + if srcIdx >= limit || srcIdx < 0 { + rg.array[regv+i] = LNil + } else { + rg.array[regv+i] = rg.array[srcIdx] + } + } + + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regv + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + cf.Base = base + cf.LocalBase = base + (cf.LocalBase - lbase + 1) + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_RETURN + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + // this section is inlined by go-inline + // source function is 'func (ls *LState) closeUpvalues(idx int) ' in '_state.go' + { + ls := L + idx := lbase + if ls.uvcache != nil { + var prev *Upvalue + for uv := ls.uvcache; uv != nil; uv = uv.next { + if uv.index >= idx { + if prev != nil { + prev.next = nil + } else { + ls.uvcache = nil + } + uv.Close() + } + prev = uv + } + } + } + nret := B - 1 + if B == 0 { + nret = reg.Top() - RA + } + n := cf.NRet + if cf.NRet == MultRet { + n = nret + } + + if L.Parent != nil && L.stack.Sp() == 1 { + // this section is inlined by go-inline + // source function is 'func copyReturnValues(L *LState, regv, start, n, b int) ' in '_vm.go' + { + regv := reg.Top() + start := RA + b := B + if b == 1 { + // this section is inlined by go-inline + // source function is 'func (rg *registry) FillNil(regm, n int) ' in '_state.go' + { + rg := L.reg + regm := regv + newSize := regm + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := 0; i < n; i++ { + rg.array[regm+i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regm + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + } else { + // this section is inlined by go-inline + // source function is 'func (rg *registry) CopyRange(regv, start, limit, n int) ' in '_state.go' + { + rg := L.reg + limit := -1 + newSize := regv + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + if limit == -1 || limit > rg.top { + limit = rg.top + } + for i := 0; i < n; i++ { + srcIdx := start + i + if srcIdx >= limit || srcIdx < 0 { + rg.array[regv+i] = LNil + } else { + rg.array[regv+i] = rg.array[srcIdx] + } + } + + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regv + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + if b > 1 && n > (b-1) { + // this section is inlined by go-inline + // source function is 'func (rg *registry) FillNil(regm, n int) ' in '_state.go' + { + rg := L.reg + regm := regv + b - 1 + n := n - (b - 1) + newSize := regm + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := 0; i < n; i++ { + rg.array[regm+i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regm + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + } + } + } + switchToParentThread(L, n, false, true) + return 1 + } + islast := baseframe == L.stack.Pop() || L.stack.IsEmpty() + // this section is inlined by go-inline + // source function is 'func copyReturnValues(L *LState, regv, start, n, b int) ' in '_vm.go' + { + regv := cf.ReturnBase + start := RA + b := B + if b == 1 { + // this section is inlined by go-inline + // source function is 'func (rg *registry) FillNil(regm, n int) ' in '_state.go' + { + rg := L.reg + regm := regv + newSize := regm + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := 0; i < n; i++ { + rg.array[regm+i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regm + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + } else { + // this section is inlined by go-inline + // source function is 'func (rg *registry) CopyRange(regv, start, limit, n int) ' in '_state.go' + { + rg := L.reg + limit := -1 + newSize := regv + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + if limit == -1 || limit > rg.top { + limit = rg.top + } + for i := 0; i < n; i++ { + srcIdx := start + i + if srcIdx >= limit || srcIdx < 0 { + rg.array[regv+i] = LNil + } else { + rg.array[regv+i] = rg.array[srcIdx] + } + } + + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regv + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + if b > 1 && n > (b-1) { + // this section is inlined by go-inline + // source function is 'func (rg *registry) FillNil(regm, n int) ' in '_state.go' + { + rg := L.reg + regm := regv + b - 1 + n := n - (b - 1) + newSize := regm + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + for i := 0; i < n; i++ { + rg.array[regm+i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regm + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + } + } + } + L.currentFrame = L.stack.Last() + if islast || L.currentFrame == nil || L.currentFrame.Fn.IsG { + return 1 + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_FORLOOP + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + if init, ok1 := reg.Get(RA).(LNumber); ok1 { + if limit, ok2 := reg.Get(RA + 1).(LNumber); ok2 { + if step, ok3 := reg.Get(RA + 2).(LNumber); ok3 { + init += step + v := LNumber(init) + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetNumber(regi int, vali LNumber) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } + } + if (step > 0 && init <= limit) || (step <= 0 && init >= limit) { + Sbx := int(inst&0x3ffff) - opMaxArgSbx //GETSBX + cf.Pc += Sbx + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetNumber(regi int, vali LNumber) ' in '_state.go' + { + rg := reg + regi := RA + 3 + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetTop(topi int) ' in '_state.go' + { + rg := reg + topi := RA + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := topi + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + oldtopi := rg.top + rg.top = topi + for i := oldtopi; i < rg.top; i++ { + rg.array[i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + if rg.top < oldtopi { + nilRange := rg.array[rg.top:oldtopi] + for i := range nilRange { + nilRange[i] = nil + } + } + //for i := rg.top; i < oldtop; i++ { + // rg.array[i] = LNil + //} + } + } + } else { + L.RaiseError("for statement step must be a number") + } + } else { + L.RaiseError("for statement limit must be a number") + } + } else { + L.RaiseError("for statement init must be a number") + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_FORPREP + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Sbx := int(inst&0x3ffff) - opMaxArgSbx //GETSBX + if init, ok1 := reg.Get(RA).(LNumber); ok1 { + if step, ok2 := reg.Get(RA + 2).(LNumber); ok2 { + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetNumber(regi int, vali LNumber) ' in '_state.go' + { + rg := reg + regi := RA + vali := LNumber(init - step) + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + L.RaiseError("for statement step must be a number") + } + } else { + L.RaiseError("for statement init must be a number") + } + cf.Pc += Sbx + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_TFORLOOP + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + C := int(inst>>9) & 0x1ff //GETC + nret := C + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetTop(topi int) ' in '_state.go' + { + rg := reg + topi := RA + 3 + 2 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := topi + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + oldtopi := rg.top + rg.top = topi + for i := oldtopi; i < rg.top; i++ { + rg.array[i] = LNil + } + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + if rg.top < oldtopi { + nilRange := rg.array[rg.top:oldtopi] + for i := range nilRange { + nilRange[i] = nil + } + } + //for i := rg.top; i < oldtop; i++ { + // rg.array[i] = LNil + //} + } + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + 3 + 2 + vali := reg.Get(RA + 2) + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + 3 + 1 + vali := reg.Get(RA + 1) + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + 3 + vali := reg.Get(RA) + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + L.callR(2, nret, RA+3) + if value := reg.Get(RA + 3); value != LNil { + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + 2 + vali := value + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + pc := cf.Fn.Proto.Code[cf.Pc] + cf.Pc += int(pc&0x3ffff) - opMaxArgSbx + } + cf.Pc++ + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_SETLIST + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + if C == 0 { + C = int(cf.Fn.Proto.Code[cf.Pc]) + cf.Pc++ + } + offset := (C - 1) * FieldsPerFlush + table := reg.Get(RA).(*LTable) + nelem := B + if B == 0 { + nelem = reg.Top() - RA - 1 + } + for i := 1; i <= nelem; i++ { + table.RawSetInt(offset+i, reg.Get(RA+i)) + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_CLOSE + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + // this section is inlined by go-inline + // source function is 'func (ls *LState) closeUpvalues(idx int) ' in '_state.go' + { + ls := L + idx := RA + if ls.uvcache != nil { + var prev *Upvalue + for uv := ls.uvcache; uv != nil; uv = uv.next { + if uv.index >= idx { + if prev != nil { + prev.next = nil + } else { + ls.uvcache = nil + } + uv.Close() + } + prev = uv + } + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_CLOSURE + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + Bx := int(inst & 0x3ffff) //GETBX + proto := cf.Fn.Proto.FunctionPrototypes[Bx] + closure := newLFunctionL(proto, cf.Fn.Env, int(proto.NumUpvalues)) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := closure + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + for i := 0; i < int(proto.NumUpvalues); i++ { + inst = cf.Fn.Proto.Code[cf.Pc] + cf.Pc++ + B := opGetArgB(inst) + switch opGetOpCode(inst) { + case OP_MOVE: + closure.Upvalues[i] = L.findUpvalue(lbase + B) + case OP_GETUPVAL: + closure.Upvalues[i] = cf.Fn.Upvalues[B] + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_VARARG + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + B := int(inst & 0x1ff) //GETB + nparams := int(cf.Fn.Proto.NumParameters) + nvarargs := cf.NArgs - nparams + if nvarargs < 0 { + nvarargs = 0 + } + nwant := B - 1 + if B == 0 { + nwant = nvarargs + } + // this section is inlined by go-inline + // source function is 'func (rg *registry) CopyRange(regv, start, limit, n int) ' in '_state.go' + { + rg := reg + regv := RA + start := cf.Base + nparams + 1 + limit := cf.LocalBase + n := nwant + newSize := regv + n + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + if limit == -1 || limit > rg.top { + limit = rg.top + } + for i := 0; i < n; i++ { + srcIdx := start + i + if srcIdx >= limit || srcIdx < 0 { + rg.array[regv+i] = LNil + } else { + rg.array[regv+i] = rg.array[srcIdx] + } + } + + // values beyond top don't need to be valid LValues, so setting them to nil is fine + // setting them to nil rather than LNil lets us invoke the golang memclr opto + oldtop := rg.top + rg.top = regv + n + if rg.top < oldtop { + nilRange := rg.array[rg.top:oldtop] + for i := range nilRange { + nilRange[i] = nil + } + } + } + return 0 + }, + func(L *LState, inst uint32, baseframe *callFrame) int { //OP_NOP + return 0 + }, + } +} + +func opArith(L *LState, inst uint32, baseframe *callFrame) int { //OP_ADD, OP_SUB, OP_MUL, OP_DIV, OP_MOD, OP_POW + reg := L.reg + cf := L.currentFrame + lbase := cf.LocalBase + A := int(inst>>18) & 0xff //GETA + RA := lbase + A + opcode := int(inst >> 26) //GETOPCODE + B := int(inst & 0x1ff) //GETB + C := int(inst>>9) & 0x1ff //GETC + lhs := L.rkValue(B) + rhs := L.rkValue(C) + v1, ok1 := lhs.(LNumber) + v2, ok2 := rhs.(LNumber) + if ok1 && ok2 { + v := numberArith(L, opcode, LNumber(v1), LNumber(v2)) + // this section is inlined by go-inline + // source function is 'func (rg *registry) SetNumber(regi int, vali LNumber) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = rg.alloc.LNumber2I(vali) + if regi >= rg.top { + rg.top = regi + 1 + } + } + } else { + v := objectArith(L, opcode, lhs, rhs) + // this section is inlined by go-inline + // source function is 'func (rg *registry) Set(regi int, vali LValue) ' in '_state.go' + { + rg := reg + regi := RA + vali := v + newSize := regi + 1 + // this section is inlined by go-inline + // source function is 'func (rg *registry) checkSize(requiredSize int) ' in '_state.go' + { + requiredSize := newSize + if requiredSize > cap(rg.array) { + rg.resize(requiredSize) + } + } + rg.array[regi] = vali + if regi >= rg.top { + rg.top = regi + 1 + } + } + } + return 0 +} + +func luaModulo(lhs, rhs LNumber) LNumber { + flhs := float64(lhs) + frhs := float64(rhs) + v := math.Mod(flhs, frhs) + if frhs > 0 && v < 0 || frhs < 0 && v > 0 { + v += frhs + } + return LNumber(v) +} + +func numberArith(L *LState, opcode int, lhs, rhs LNumber) LNumber { + switch opcode { + case OP_ADD: + return lhs + rhs + case OP_SUB: + return lhs - rhs + case OP_MUL: + return lhs * rhs + case OP_DIV: + return lhs / rhs + case OP_MOD: + return luaModulo(lhs, rhs) + case OP_POW: + flhs := float64(lhs) + frhs := float64(rhs) + return LNumber(math.Pow(flhs, frhs)) + } + panic("should not reach here") + return LNumber(0) +} + +func objectArith(L *LState, opcode int, lhs, rhs LValue) LValue { + event := "" + switch opcode { + case OP_ADD: + event = "__add" + case OP_SUB: + event = "__sub" + case OP_MUL: + event = "__mul" + case OP_DIV: + event = "__div" + case OP_MOD: + event = "__mod" + case OP_POW: + event = "__pow" + } + op := L.metaOp2(lhs, rhs, event) + if _, ok := op.(*LFunction); ok { + L.reg.Push(op) + L.reg.Push(lhs) + L.reg.Push(rhs) + L.Call(2, 1) + return L.reg.Pop() + } + if str, ok := lhs.(LString); ok { + if lnum, err := parseNumber(string(str)); err == nil { + lhs = lnum + } + } + if str, ok := rhs.(LString); ok { + if rnum, err := parseNumber(string(str)); err == nil { + rhs = rnum + } + } + if v1, ok1 := lhs.(LNumber); ok1 { + if v2, ok2 := rhs.(LNumber); ok2 { + return numberArith(L, opcode, LNumber(v1), LNumber(v2)) + } + } + L.RaiseError(fmt.Sprintf("cannot perform %v operation between %v and %v", + strings.TrimLeft(event, "_"), lhs.Type().String(), rhs.Type().String())) + + return LNil +} + +func stringConcat(L *LState, total, last int) LValue { + rhs := L.reg.Get(last) + total-- + for i := last - 1; total > 0; { + lhs := L.reg.Get(i) + if !(LVCanConvToString(lhs) && LVCanConvToString(rhs)) { + op := L.metaOp2(lhs, rhs, "__concat") + if op.Type() == LTFunction { + L.reg.Push(op) + L.reg.Push(lhs) + L.reg.Push(rhs) + L.Call(2, 1) + rhs = L.reg.Pop() + total-- + i-- + } else { + L.RaiseError("cannot perform concat operation between %v and %v", lhs.Type().String(), rhs.Type().String()) + return LNil + } + } else { + buf := make([]string, total+1) + buf[total] = LVAsString(rhs) + for total > 0 { + lhs = L.reg.Get(i) + if !LVCanConvToString(lhs) { + break + } + buf[total-1] = LVAsString(lhs) + i-- + total-- + } + rhs = LString(strings.Join(buf, "")) + } + } + return rhs +} + +func lessThan(L *LState, lhs, rhs LValue) bool { + // optimization for numbers + if v1, ok1 := lhs.(LNumber); ok1 { + if v2, ok2 := rhs.(LNumber); ok2 { + return v1 < v2 + } + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + } + if lhs.Type() != rhs.Type() { + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + return false + } + ret := false + switch lhs.Type() { + case LTString: + ret = strCmp(string(lhs.(LString)), string(rhs.(LString))) < 0 + default: + ret = objectRationalWithError(L, lhs, rhs, "__lt") + } + return ret +} + +func equals(L *LState, lhs, rhs LValue, raw bool) bool { + lt := lhs.Type() + if lt != rhs.Type() { + return false + } + + ret := false + switch lt { + case LTNil: + ret = true + case LTNumber: + v1, _ := lhs.(LNumber) + v2, _ := rhs.(LNumber) + ret = v1 == v2 + case LTBool: + ret = bool(lhs.(LBool)) == bool(rhs.(LBool)) + case LTString: + ret = string(lhs.(LString)) == string(rhs.(LString)) + case LTUserData, LTTable: + if lhs == rhs { + ret = true + } else if !raw { + switch objectRational(L, lhs, rhs, "__eq") { + case 1: + ret = true + default: + ret = false + } + } + default: + ret = lhs == rhs + } + return ret +} + +func objectRationalWithError(L *LState, lhs, rhs LValue, event string) bool { + switch objectRational(L, lhs, rhs, event) { + case 1: + return true + case 0: + return false + } + L.RaiseError("attempt to compare %v with %v", lhs.Type().String(), rhs.Type().String()) + return false +} + +func objectRational(L *LState, lhs, rhs LValue, event string) int { + m1 := L.metaOp1(lhs, event) + m2 := L.metaOp1(rhs, event) + if m1.Type() == LTFunction && m1 == m2 { + L.reg.Push(m1) + L.reg.Push(lhs) + L.reg.Push(rhs) + L.Call(2, 1) + if LVAsBool(L.reg.Pop()) { + return 1 + } + return 0 + } + return -1 +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/LICENSE b/scraper-go/vendor/gopkg.in/yaml.v3/LICENSE new file mode 100644 index 00000000..2683e4bb --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/LICENSE @@ -0,0 +1,50 @@ + +This project is covered by two different licenses: MIT and Apache. + +#### MIT License #### + +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original MIT license, with the additional +copyright staring in 2011 when the project was ported over: + + apic.go emitterc.go parserc.go readerc.go scannerc.go + writerc.go yamlh.go yamlprivateh.go + +Copyright (c) 2006-2010 Kirill Simonov +Copyright (c) 2006-2011 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +### Apache License ### + +All the remaining project files are covered by the Apache license: + +Copyright (c) 2011-2019 Canonical Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/NOTICE b/scraper-go/vendor/gopkg.in/yaml.v3/NOTICE new file mode 100644 index 00000000..866d74a7 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/NOTICE @@ -0,0 +1,13 @@ +Copyright 2011-2016 Canonical Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/README.md b/scraper-go/vendor/gopkg.in/yaml.v3/README.md new file mode 100644 index 00000000..08eb1bab --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/README.md @@ -0,0 +1,150 @@ +# YAML support for the Go language + +Introduction +------------ + +The yaml package enables Go programs to comfortably encode and decode YAML +values. It was developed within [Canonical](https://www.canonical.com) as +part of the [juju](https://juju.ubuntu.com) project, and is based on a +pure Go port of the well-known [libyaml](http://pyyaml.org/wiki/LibYAML) +C library to parse and generate YAML data quickly and reliably. + +Compatibility +------------- + +The yaml package supports most of YAML 1.2, but preserves some behavior +from 1.1 for backwards compatibility. + +Specifically, as of v3 of the yaml package: + + - YAML 1.1 bools (_yes/no, on/off_) are supported as long as they are being + decoded into a typed bool value. Otherwise they behave as a string. Booleans + in YAML 1.2 are _true/false_ only. + - Octals encode and decode as _0777_ per YAML 1.1, rather than _0o777_ + as specified in YAML 1.2, because most parsers still use the old format. + Octals in the _0o777_ format are supported though, so new files work. + - Does not support base-60 floats. These are gone from YAML 1.2, and were + actually never supported by this package as it's clearly a poor choice. + +and offers backwards +compatibility with YAML 1.1 in some cases. +1.2, including support for +anchors, tags, map merging, etc. Multi-document unmarshalling is not yet +implemented, and base-60 floats from YAML 1.1 are purposefully not +supported since they're a poor design and are gone in YAML 1.2. + +Installation and usage +---------------------- + +The import path for the package is *gopkg.in/yaml.v3*. + +To install it, run: + + go get gopkg.in/yaml.v3 + +API documentation +----------------- + +If opened in a browser, the import path itself leads to the API documentation: + + - [https://gopkg.in/yaml.v3](https://gopkg.in/yaml.v3) + +API stability +------------- + +The package API for yaml v3 will remain stable as described in [gopkg.in](https://gopkg.in). + + +License +------- + +The yaml package is licensed under the MIT and Apache License 2.0 licenses. +Please see the LICENSE file for details. + + +Example +------- + +```Go +package main + +import ( + "fmt" + "log" + + "gopkg.in/yaml.v3" +) + +var data = ` +a: Easy! +b: + c: 2 + d: [3, 4] +` + +// Note: struct fields must be public in order for unmarshal to +// correctly populate the data. +type T struct { + A string + B struct { + RenamedC int `yaml:"c"` + D []int `yaml:",flow"` + } +} + +func main() { + t := T{} + + err := yaml.Unmarshal([]byte(data), &t) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- t:\n%v\n\n", t) + + d, err := yaml.Marshal(&t) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- t dump:\n%s\n\n", string(d)) + + m := make(map[interface{}]interface{}) + + err = yaml.Unmarshal([]byte(data), &m) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- m:\n%v\n\n", m) + + d, err = yaml.Marshal(&m) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- m dump:\n%s\n\n", string(d)) +} +``` + +This example will generate the following output: + +``` +--- t: +{Easy! {2 [3 4]}} + +--- t dump: +a: Easy! +b: + c: 2 + d: [3, 4] + + +--- m: +map[a:Easy! b:map[c:2 d:[3 4]]] + +--- m dump: +a: Easy! +b: + c: 2 + d: + - 3 + - 4 +``` + diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/apic.go b/scraper-go/vendor/gopkg.in/yaml.v3/apic.go new file mode 100644 index 00000000..ae7d049f --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/apic.go @@ -0,0 +1,747 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "io" +) + +func yaml_insert_token(parser *yaml_parser_t, pos int, token *yaml_token_t) { + //fmt.Println("yaml_insert_token", "pos:", pos, "typ:", token.typ, "head:", parser.tokens_head, "len:", len(parser.tokens)) + + // Check if we can move the queue at the beginning of the buffer. + if parser.tokens_head > 0 && len(parser.tokens) == cap(parser.tokens) { + if parser.tokens_head != len(parser.tokens) { + copy(parser.tokens, parser.tokens[parser.tokens_head:]) + } + parser.tokens = parser.tokens[:len(parser.tokens)-parser.tokens_head] + parser.tokens_head = 0 + } + parser.tokens = append(parser.tokens, *token) + if pos < 0 { + return + } + copy(parser.tokens[parser.tokens_head+pos+1:], parser.tokens[parser.tokens_head+pos:]) + parser.tokens[parser.tokens_head+pos] = *token +} + +// Create a new parser object. +func yaml_parser_initialize(parser *yaml_parser_t) bool { + *parser = yaml_parser_t{ + raw_buffer: make([]byte, 0, input_raw_buffer_size), + buffer: make([]byte, 0, input_buffer_size), + } + return true +} + +// Destroy a parser object. +func yaml_parser_delete(parser *yaml_parser_t) { + *parser = yaml_parser_t{} +} + +// String read handler. +func yaml_string_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + if parser.input_pos == len(parser.input) { + return 0, io.EOF + } + n = copy(buffer, parser.input[parser.input_pos:]) + parser.input_pos += n + return n, nil +} + +// Reader read handler. +func yaml_reader_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + return parser.input_reader.Read(buffer) +} + +// Set a string input. +func yaml_parser_set_input_string(parser *yaml_parser_t, input []byte) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_string_read_handler + parser.input = input + parser.input_pos = 0 +} + +// Set a file input. +func yaml_parser_set_input_reader(parser *yaml_parser_t, r io.Reader) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_reader_read_handler + parser.input_reader = r +} + +// Set the source encoding. +func yaml_parser_set_encoding(parser *yaml_parser_t, encoding yaml_encoding_t) { + if parser.encoding != yaml_ANY_ENCODING { + panic("must set the encoding only once") + } + parser.encoding = encoding +} + +// Create a new emitter object. +func yaml_emitter_initialize(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{ + buffer: make([]byte, output_buffer_size), + raw_buffer: make([]byte, 0, output_raw_buffer_size), + states: make([]yaml_emitter_state_t, 0, initial_stack_size), + events: make([]yaml_event_t, 0, initial_queue_size), + best_width: -1, + } +} + +// Destroy an emitter object. +func yaml_emitter_delete(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{} +} + +// String write handler. +func yaml_string_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + *emitter.output_buffer = append(*emitter.output_buffer, buffer...) + return nil +} + +// yaml_writer_write_handler uses emitter.output_writer to write the +// emitted text. +func yaml_writer_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + _, err := emitter.output_writer.Write(buffer) + return err +} + +// Set a string output. +func yaml_emitter_set_output_string(emitter *yaml_emitter_t, output_buffer *[]byte) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_string_write_handler + emitter.output_buffer = output_buffer +} + +// Set a file output. +func yaml_emitter_set_output_writer(emitter *yaml_emitter_t, w io.Writer) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_writer_write_handler + emitter.output_writer = w +} + +// Set the output encoding. +func yaml_emitter_set_encoding(emitter *yaml_emitter_t, encoding yaml_encoding_t) { + if emitter.encoding != yaml_ANY_ENCODING { + panic("must set the output encoding only once") + } + emitter.encoding = encoding +} + +// Set the canonical output style. +func yaml_emitter_set_canonical(emitter *yaml_emitter_t, canonical bool) { + emitter.canonical = canonical +} + +// Set the indentation increment. +func yaml_emitter_set_indent(emitter *yaml_emitter_t, indent int) { + if indent < 2 || indent > 9 { + indent = 2 + } + emitter.best_indent = indent +} + +// Set the preferred line width. +func yaml_emitter_set_width(emitter *yaml_emitter_t, width int) { + if width < 0 { + width = -1 + } + emitter.best_width = width +} + +// Set if unescaped non-ASCII characters are allowed. +func yaml_emitter_set_unicode(emitter *yaml_emitter_t, unicode bool) { + emitter.unicode = unicode +} + +// Set the preferred line break character. +func yaml_emitter_set_break(emitter *yaml_emitter_t, line_break yaml_break_t) { + emitter.line_break = line_break +} + +///* +// * Destroy a token object. +// */ +// +//YAML_DECLARE(void) +//yaml_token_delete(yaml_token_t *token) +//{ +// assert(token); // Non-NULL token object expected. +// +// switch (token.type) +// { +// case YAML_TAG_DIRECTIVE_TOKEN: +// yaml_free(token.data.tag_directive.handle); +// yaml_free(token.data.tag_directive.prefix); +// break; +// +// case YAML_ALIAS_TOKEN: +// yaml_free(token.data.alias.value); +// break; +// +// case YAML_ANCHOR_TOKEN: +// yaml_free(token.data.anchor.value); +// break; +// +// case YAML_TAG_TOKEN: +// yaml_free(token.data.tag.handle); +// yaml_free(token.data.tag.suffix); +// break; +// +// case YAML_SCALAR_TOKEN: +// yaml_free(token.data.scalar.value); +// break; +// +// default: +// break; +// } +// +// memset(token, 0, sizeof(yaml_token_t)); +//} +// +///* +// * Check if a string is a valid UTF-8 sequence. +// * +// * Check 'reader.c' for more details on UTF-8 encoding. +// */ +// +//static int +//yaml_check_utf8(yaml_char_t *start, size_t length) +//{ +// yaml_char_t *end = start+length; +// yaml_char_t *pointer = start; +// +// while (pointer < end) { +// unsigned char octet; +// unsigned int width; +// unsigned int value; +// size_t k; +// +// octet = pointer[0]; +// width = (octet & 0x80) == 0x00 ? 1 : +// (octet & 0xE0) == 0xC0 ? 2 : +// (octet & 0xF0) == 0xE0 ? 3 : +// (octet & 0xF8) == 0xF0 ? 4 : 0; +// value = (octet & 0x80) == 0x00 ? octet & 0x7F : +// (octet & 0xE0) == 0xC0 ? octet & 0x1F : +// (octet & 0xF0) == 0xE0 ? octet & 0x0F : +// (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; +// if (!width) return 0; +// if (pointer+width > end) return 0; +// for (k = 1; k < width; k ++) { +// octet = pointer[k]; +// if ((octet & 0xC0) != 0x80) return 0; +// value = (value << 6) + (octet & 0x3F); +// } +// if (!((width == 1) || +// (width == 2 && value >= 0x80) || +// (width == 3 && value >= 0x800) || +// (width == 4 && value >= 0x10000))) return 0; +// +// pointer += width; +// } +// +// return 1; +//} +// + +// Create STREAM-START. +func yaml_stream_start_event_initialize(event *yaml_event_t, encoding yaml_encoding_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + encoding: encoding, + } +} + +// Create STREAM-END. +func yaml_stream_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + } +} + +// Create DOCUMENT-START. +func yaml_document_start_event_initialize( + event *yaml_event_t, + version_directive *yaml_version_directive_t, + tag_directives []yaml_tag_directive_t, + implicit bool, +) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: implicit, + } +} + +// Create DOCUMENT-END. +func yaml_document_end_event_initialize(event *yaml_event_t, implicit bool) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + implicit: implicit, + } +} + +// Create ALIAS. +func yaml_alias_event_initialize(event *yaml_event_t, anchor []byte) bool { + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + anchor: anchor, + } + return true +} + +// Create SCALAR. +func yaml_scalar_event_initialize(event *yaml_event_t, anchor, tag, value []byte, plain_implicit, quoted_implicit bool, style yaml_scalar_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + anchor: anchor, + tag: tag, + value: value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-START. +func yaml_sequence_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_sequence_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-END. +func yaml_sequence_end_event_initialize(event *yaml_event_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + } + return true +} + +// Create MAPPING-START. +func yaml_mapping_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_mapping_style_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } +} + +// Create MAPPING-END. +func yaml_mapping_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + } +} + +// Destroy an event object. +func yaml_event_delete(event *yaml_event_t) { + *event = yaml_event_t{} +} + +///* +// * Create a document object. +// */ +// +//YAML_DECLARE(int) +//yaml_document_initialize(document *yaml_document_t, +// version_directive *yaml_version_directive_t, +// tag_directives_start *yaml_tag_directive_t, +// tag_directives_end *yaml_tag_directive_t, +// start_implicit int, end_implicit int) +//{ +// struct { +// error yaml_error_type_t +// } context +// struct { +// start *yaml_node_t +// end *yaml_node_t +// top *yaml_node_t +// } nodes = { NULL, NULL, NULL } +// version_directive_copy *yaml_version_directive_t = NULL +// struct { +// start *yaml_tag_directive_t +// end *yaml_tag_directive_t +// top *yaml_tag_directive_t +// } tag_directives_copy = { NULL, NULL, NULL } +// value yaml_tag_directive_t = { NULL, NULL } +// mark yaml_mark_t = { 0, 0, 0 } +// +// assert(document) // Non-NULL document object is expected. +// assert((tag_directives_start && tag_directives_end) || +// (tag_directives_start == tag_directives_end)) +// // Valid tag directives are expected. +// +// if (!STACK_INIT(&context, nodes, INITIAL_STACK_SIZE)) goto error +// +// if (version_directive) { +// version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)) +// if (!version_directive_copy) goto error +// version_directive_copy.major = version_directive.major +// version_directive_copy.minor = version_directive.minor +// } +// +// if (tag_directives_start != tag_directives_end) { +// tag_directive *yaml_tag_directive_t +// if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) +// goto error +// for (tag_directive = tag_directives_start +// tag_directive != tag_directives_end; tag_directive ++) { +// assert(tag_directive.handle) +// assert(tag_directive.prefix) +// if (!yaml_check_utf8(tag_directive.handle, +// strlen((char *)tag_directive.handle))) +// goto error +// if (!yaml_check_utf8(tag_directive.prefix, +// strlen((char *)tag_directive.prefix))) +// goto error +// value.handle = yaml_strdup(tag_directive.handle) +// value.prefix = yaml_strdup(tag_directive.prefix) +// if (!value.handle || !value.prefix) goto error +// if (!PUSH(&context, tag_directives_copy, value)) +// goto error +// value.handle = NULL +// value.prefix = NULL +// } +// } +// +// DOCUMENT_INIT(*document, nodes.start, nodes.end, version_directive_copy, +// tag_directives_copy.start, tag_directives_copy.top, +// start_implicit, end_implicit, mark, mark) +// +// return 1 +// +//error: +// STACK_DEL(&context, nodes) +// yaml_free(version_directive_copy) +// while (!STACK_EMPTY(&context, tag_directives_copy)) { +// value yaml_tag_directive_t = POP(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// } +// STACK_DEL(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// +// return 0 +//} +// +///* +// * Destroy a document object. +// */ +// +//YAML_DECLARE(void) +//yaml_document_delete(document *yaml_document_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// tag_directive *yaml_tag_directive_t +// +// context.error = YAML_NO_ERROR // Eliminate a compiler warning. +// +// assert(document) // Non-NULL document object is expected. +// +// while (!STACK_EMPTY(&context, document.nodes)) { +// node yaml_node_t = POP(&context, document.nodes) +// yaml_free(node.tag) +// switch (node.type) { +// case YAML_SCALAR_NODE: +// yaml_free(node.data.scalar.value) +// break +// case YAML_SEQUENCE_NODE: +// STACK_DEL(&context, node.data.sequence.items) +// break +// case YAML_MAPPING_NODE: +// STACK_DEL(&context, node.data.mapping.pairs) +// break +// default: +// assert(0) // Should not happen. +// } +// } +// STACK_DEL(&context, document.nodes) +// +// yaml_free(document.version_directive) +// for (tag_directive = document.tag_directives.start +// tag_directive != document.tag_directives.end +// tag_directive++) { +// yaml_free(tag_directive.handle) +// yaml_free(tag_directive.prefix) +// } +// yaml_free(document.tag_directives.start) +// +// memset(document, 0, sizeof(yaml_document_t)) +//} +// +///** +// * Get a document node. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_node(document *yaml_document_t, index int) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (index > 0 && document.nodes.start + index <= document.nodes.top) { +// return document.nodes.start + index - 1 +// } +// return NULL +//} +// +///** +// * Get the root object. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_root_node(document *yaml_document_t) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (document.nodes.top != document.nodes.start) { +// return document.nodes.start +// } +// return NULL +//} +// +///* +// * Add a scalar node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_scalar(document *yaml_document_t, +// tag *yaml_char_t, value *yaml_char_t, length int, +// style yaml_scalar_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// value_copy *yaml_char_t = NULL +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// assert(value) // Non-NULL value is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (length < 0) { +// length = strlen((char *)value) +// } +// +// if (!yaml_check_utf8(value, length)) goto error +// value_copy = yaml_malloc(length+1) +// if (!value_copy) goto error +// memcpy(value_copy, value, length) +// value_copy[length] = '\0' +// +// SCALAR_NODE_INIT(node, tag_copy, value_copy, length, style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// yaml_free(tag_copy) +// yaml_free(value_copy) +// +// return 0 +//} +// +///* +// * Add a sequence node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_sequence(document *yaml_document_t, +// tag *yaml_char_t, style yaml_sequence_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_item_t +// end *yaml_node_item_t +// top *yaml_node_item_t +// } items = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, items, INITIAL_STACK_SIZE)) goto error +// +// SEQUENCE_NODE_INIT(node, tag_copy, items.start, items.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, items) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Add a mapping node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_mapping(document *yaml_document_t, +// tag *yaml_char_t, style yaml_mapping_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_pair_t +// end *yaml_node_pair_t +// top *yaml_node_pair_t +// } pairs = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, pairs, INITIAL_STACK_SIZE)) goto error +// +// MAPPING_NODE_INIT(node, tag_copy, pairs.start, pairs.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, pairs) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Append an item to a sequence node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_sequence_item(document *yaml_document_t, +// sequence int, item int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// assert(document) // Non-NULL document is required. +// assert(sequence > 0 +// && document.nodes.start + sequence <= document.nodes.top) +// // Valid sequence id is required. +// assert(document.nodes.start[sequence-1].type == YAML_SEQUENCE_NODE) +// // A sequence node is required. +// assert(item > 0 && document.nodes.start + item <= document.nodes.top) +// // Valid item id is required. +// +// if (!PUSH(&context, +// document.nodes.start[sequence-1].data.sequence.items, item)) +// return 0 +// +// return 1 +//} +// +///* +// * Append a pair of a key and a value to a mapping node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_mapping_pair(document *yaml_document_t, +// mapping int, key int, value int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// pair yaml_node_pair_t +// +// assert(document) // Non-NULL document is required. +// assert(mapping > 0 +// && document.nodes.start + mapping <= document.nodes.top) +// // Valid mapping id is required. +// assert(document.nodes.start[mapping-1].type == YAML_MAPPING_NODE) +// // A mapping node is required. +// assert(key > 0 && document.nodes.start + key <= document.nodes.top) +// // Valid key id is required. +// assert(value > 0 && document.nodes.start + value <= document.nodes.top) +// // Valid value id is required. +// +// pair.key = key +// pair.value = value +// +// if (!PUSH(&context, +// document.nodes.start[mapping-1].data.mapping.pairs, pair)) +// return 0 +// +// return 1 +//} +// +// diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/decode.go b/scraper-go/vendor/gopkg.in/yaml.v3/decode.go new file mode 100644 index 00000000..0173b698 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/decode.go @@ -0,0 +1,1000 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "encoding" + "encoding/base64" + "fmt" + "io" + "math" + "reflect" + "strconv" + "time" +) + +// ---------------------------------------------------------------------------- +// Parser, produces a node tree out of a libyaml event stream. + +type parser struct { + parser yaml_parser_t + event yaml_event_t + doc *Node + anchors map[string]*Node + doneInit bool + textless bool +} + +func newParser(b []byte) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + if len(b) == 0 { + b = []byte{'\n'} + } + yaml_parser_set_input_string(&p.parser, b) + return &p +} + +func newParserFromReader(r io.Reader) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + yaml_parser_set_input_reader(&p.parser, r) + return &p +} + +func (p *parser) init() { + if p.doneInit { + return + } + p.anchors = make(map[string]*Node) + p.expect(yaml_STREAM_START_EVENT) + p.doneInit = true +} + +func (p *parser) destroy() { + if p.event.typ != yaml_NO_EVENT { + yaml_event_delete(&p.event) + } + yaml_parser_delete(&p.parser) +} + +// expect consumes an event from the event stream and +// checks that it's of the expected type. +func (p *parser) expect(e yaml_event_type_t) { + if p.event.typ == yaml_NO_EVENT { + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + } + if p.event.typ == yaml_STREAM_END_EVENT { + failf("attempted to go past the end of stream; corrupted value?") + } + if p.event.typ != e { + p.parser.problem = fmt.Sprintf("expected %s event but got %s", e, p.event.typ) + p.fail() + } + yaml_event_delete(&p.event) + p.event.typ = yaml_NO_EVENT +} + +// peek peeks at the next event in the event stream, +// puts the results into p.event and returns the event type. +func (p *parser) peek() yaml_event_type_t { + if p.event.typ != yaml_NO_EVENT { + return p.event.typ + } + // It's curious choice from the underlying API to generally return a + // positive result on success, but on this case return true in an error + // scenario. This was the source of bugs in the past (issue #666). + if !yaml_parser_parse(&p.parser, &p.event) || p.parser.error != yaml_NO_ERROR { + p.fail() + } + return p.event.typ +} + +func (p *parser) fail() { + var where string + var line int + if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } else if p.parser.problem_mark.line != 0 { + line = p.parser.problem_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } + if line != 0 { + where = "line " + strconv.Itoa(line) + ": " + } + var msg string + if len(p.parser.problem) > 0 { + msg = p.parser.problem + } else { + msg = "unknown problem parsing YAML content" + } + failf("%s%s", where, msg) +} + +func (p *parser) anchor(n *Node, anchor []byte) { + if anchor != nil { + n.Anchor = string(anchor) + p.anchors[n.Anchor] = n + } +} + +func (p *parser) parse() *Node { + p.init() + switch p.peek() { + case yaml_SCALAR_EVENT: + return p.scalar() + case yaml_ALIAS_EVENT: + return p.alias() + case yaml_MAPPING_START_EVENT: + return p.mapping() + case yaml_SEQUENCE_START_EVENT: + return p.sequence() + case yaml_DOCUMENT_START_EVENT: + return p.document() + case yaml_STREAM_END_EVENT: + // Happens when attempting to decode an empty buffer. + return nil + case yaml_TAIL_COMMENT_EVENT: + panic("internal error: unexpected tail comment event (please report)") + default: + panic("internal error: attempted to parse unknown event (please report): " + p.event.typ.String()) + } +} + +func (p *parser) node(kind Kind, defaultTag, tag, value string) *Node { + var style Style + if tag != "" && tag != "!" { + tag = shortTag(tag) + style = TaggedStyle + } else if defaultTag != "" { + tag = defaultTag + } else if kind == ScalarNode { + tag, _ = resolve("", value) + } + n := &Node{ + Kind: kind, + Tag: tag, + Value: value, + Style: style, + } + if !p.textless { + n.Line = p.event.start_mark.line + 1 + n.Column = p.event.start_mark.column + 1 + n.HeadComment = string(p.event.head_comment) + n.LineComment = string(p.event.line_comment) + n.FootComment = string(p.event.foot_comment) + } + return n +} + +func (p *parser) parseChild(parent *Node) *Node { + child := p.parse() + parent.Content = append(parent.Content, child) + return child +} + +func (p *parser) document() *Node { + n := p.node(DocumentNode, "", "", "") + p.doc = n + p.expect(yaml_DOCUMENT_START_EVENT) + p.parseChild(n) + if p.peek() == yaml_DOCUMENT_END_EVENT { + n.FootComment = string(p.event.foot_comment) + } + p.expect(yaml_DOCUMENT_END_EVENT) + return n +} + +func (p *parser) alias() *Node { + n := p.node(AliasNode, "", "", string(p.event.anchor)) + n.Alias = p.anchors[n.Value] + if n.Alias == nil { + failf("unknown anchor '%s' referenced", n.Value) + } + p.expect(yaml_ALIAS_EVENT) + return n +} + +func (p *parser) scalar() *Node { + var parsedStyle = p.event.scalar_style() + var nodeStyle Style + switch { + case parsedStyle&yaml_DOUBLE_QUOTED_SCALAR_STYLE != 0: + nodeStyle = DoubleQuotedStyle + case parsedStyle&yaml_SINGLE_QUOTED_SCALAR_STYLE != 0: + nodeStyle = SingleQuotedStyle + case parsedStyle&yaml_LITERAL_SCALAR_STYLE != 0: + nodeStyle = LiteralStyle + case parsedStyle&yaml_FOLDED_SCALAR_STYLE != 0: + nodeStyle = FoldedStyle + } + var nodeValue = string(p.event.value) + var nodeTag = string(p.event.tag) + var defaultTag string + if nodeStyle == 0 { + if nodeValue == "<<" { + defaultTag = mergeTag + } + } else { + defaultTag = strTag + } + n := p.node(ScalarNode, defaultTag, nodeTag, nodeValue) + n.Style |= nodeStyle + p.anchor(n, p.event.anchor) + p.expect(yaml_SCALAR_EVENT) + return n +} + +func (p *parser) sequence() *Node { + n := p.node(SequenceNode, seqTag, string(p.event.tag), "") + if p.event.sequence_style()&yaml_FLOW_SEQUENCE_STYLE != 0 { + n.Style |= FlowStyle + } + p.anchor(n, p.event.anchor) + p.expect(yaml_SEQUENCE_START_EVENT) + for p.peek() != yaml_SEQUENCE_END_EVENT { + p.parseChild(n) + } + n.LineComment = string(p.event.line_comment) + n.FootComment = string(p.event.foot_comment) + p.expect(yaml_SEQUENCE_END_EVENT) + return n +} + +func (p *parser) mapping() *Node { + n := p.node(MappingNode, mapTag, string(p.event.tag), "") + block := true + if p.event.mapping_style()&yaml_FLOW_MAPPING_STYLE != 0 { + block = false + n.Style |= FlowStyle + } + p.anchor(n, p.event.anchor) + p.expect(yaml_MAPPING_START_EVENT) + for p.peek() != yaml_MAPPING_END_EVENT { + k := p.parseChild(n) + if block && k.FootComment != "" { + // Must be a foot comment for the prior value when being dedented. + if len(n.Content) > 2 { + n.Content[len(n.Content)-3].FootComment = k.FootComment + k.FootComment = "" + } + } + v := p.parseChild(n) + if k.FootComment == "" && v.FootComment != "" { + k.FootComment = v.FootComment + v.FootComment = "" + } + if p.peek() == yaml_TAIL_COMMENT_EVENT { + if k.FootComment == "" { + k.FootComment = string(p.event.foot_comment) + } + p.expect(yaml_TAIL_COMMENT_EVENT) + } + } + n.LineComment = string(p.event.line_comment) + n.FootComment = string(p.event.foot_comment) + if n.Style&FlowStyle == 0 && n.FootComment != "" && len(n.Content) > 1 { + n.Content[len(n.Content)-2].FootComment = n.FootComment + n.FootComment = "" + } + p.expect(yaml_MAPPING_END_EVENT) + return n +} + +// ---------------------------------------------------------------------------- +// Decoder, unmarshals a node into a provided value. + +type decoder struct { + doc *Node + aliases map[*Node]bool + terrors []string + + stringMapType reflect.Type + generalMapType reflect.Type + + knownFields bool + uniqueKeys bool + decodeCount int + aliasCount int + aliasDepth int + + mergedFields map[interface{}]bool +} + +var ( + nodeType = reflect.TypeOf(Node{}) + durationType = reflect.TypeOf(time.Duration(0)) + stringMapType = reflect.TypeOf(map[string]interface{}{}) + generalMapType = reflect.TypeOf(map[interface{}]interface{}{}) + ifaceType = generalMapType.Elem() + timeType = reflect.TypeOf(time.Time{}) + ptrTimeType = reflect.TypeOf(&time.Time{}) +) + +func newDecoder() *decoder { + d := &decoder{ + stringMapType: stringMapType, + generalMapType: generalMapType, + uniqueKeys: true, + } + d.aliases = make(map[*Node]bool) + return d +} + +func (d *decoder) terror(n *Node, tag string, out reflect.Value) { + if n.Tag != "" { + tag = n.Tag + } + value := n.Value + if tag != seqTag && tag != mapTag { + if len(value) > 10 { + value = " `" + value[:7] + "...`" + } else { + value = " `" + value + "`" + } + } + d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.Line, shortTag(tag), value, out.Type())) +} + +func (d *decoder) callUnmarshaler(n *Node, u Unmarshaler) (good bool) { + err := u.UnmarshalYAML(n) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +func (d *decoder) callObsoleteUnmarshaler(n *Node, u obsoleteUnmarshaler) (good bool) { + terrlen := len(d.terrors) + err := u.UnmarshalYAML(func(v interface{}) (err error) { + defer handleErr(&err) + d.unmarshal(n, reflect.ValueOf(v)) + if len(d.terrors) > terrlen { + issues := d.terrors[terrlen:] + d.terrors = d.terrors[:terrlen] + return &TypeError{issues} + } + return nil + }) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +// d.prepare initializes and dereferences pointers and calls UnmarshalYAML +// if a value is found to implement it. +// It returns the initialized and dereferenced out value, whether +// unmarshalling was already done by UnmarshalYAML, and if so whether +// its types unmarshalled appropriately. +// +// If n holds a null value, prepare returns before doing anything. +func (d *decoder) prepare(n *Node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { + if n.ShortTag() == nullTag { + return out, false, false + } + again := true + for again { + again = false + if out.Kind() == reflect.Ptr { + if out.IsNil() { + out.Set(reflect.New(out.Type().Elem())) + } + out = out.Elem() + again = true + } + if out.CanAddr() { + outi := out.Addr().Interface() + if u, ok := outi.(Unmarshaler); ok { + good = d.callUnmarshaler(n, u) + return out, true, good + } + if u, ok := outi.(obsoleteUnmarshaler); ok { + good = d.callObsoleteUnmarshaler(n, u) + return out, true, good + } + } + } + return out, false, false +} + +func (d *decoder) fieldByIndex(n *Node, v reflect.Value, index []int) (field reflect.Value) { + if n.ShortTag() == nullTag { + return reflect.Value{} + } + for _, num := range index { + for { + if v.Kind() == reflect.Ptr { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + continue + } + break + } + v = v.Field(num) + } + return v +} + +const ( + // 400,000 decode operations is ~500kb of dense object declarations, or + // ~5kb of dense object declarations with 10000% alias expansion + alias_ratio_range_low = 400000 + + // 4,000,000 decode operations is ~5MB of dense object declarations, or + // ~4.5MB of dense object declarations with 10% alias expansion + alias_ratio_range_high = 4000000 + + // alias_ratio_range is the range over which we scale allowed alias ratios + alias_ratio_range = float64(alias_ratio_range_high - alias_ratio_range_low) +) + +func allowedAliasRatio(decodeCount int) float64 { + switch { + case decodeCount <= alias_ratio_range_low: + // allow 99% to come from alias expansion for small-to-medium documents + return 0.99 + case decodeCount >= alias_ratio_range_high: + // allow 10% to come from alias expansion for very large documents + return 0.10 + default: + // scale smoothly from 99% down to 10% over the range. + // this maps to 396,000 - 400,000 allowed alias-driven decodes over the range. + // 400,000 decode operations is ~100MB of allocations in worst-case scenarios (single-item maps). + return 0.99 - 0.89*(float64(decodeCount-alias_ratio_range_low)/alias_ratio_range) + } +} + +func (d *decoder) unmarshal(n *Node, out reflect.Value) (good bool) { + d.decodeCount++ + if d.aliasDepth > 0 { + d.aliasCount++ + } + if d.aliasCount > 100 && d.decodeCount > 1000 && float64(d.aliasCount)/float64(d.decodeCount) > allowedAliasRatio(d.decodeCount) { + failf("document contains excessive aliasing") + } + if out.Type() == nodeType { + out.Set(reflect.ValueOf(n).Elem()) + return true + } + switch n.Kind { + case DocumentNode: + return d.document(n, out) + case AliasNode: + return d.alias(n, out) + } + out, unmarshaled, good := d.prepare(n, out) + if unmarshaled { + return good + } + switch n.Kind { + case ScalarNode: + good = d.scalar(n, out) + case MappingNode: + good = d.mapping(n, out) + case SequenceNode: + good = d.sequence(n, out) + case 0: + if n.IsZero() { + return d.null(out) + } + fallthrough + default: + failf("cannot decode node with unknown kind %d", n.Kind) + } + return good +} + +func (d *decoder) document(n *Node, out reflect.Value) (good bool) { + if len(n.Content) == 1 { + d.doc = n + d.unmarshal(n.Content[0], out) + return true + } + return false +} + +func (d *decoder) alias(n *Node, out reflect.Value) (good bool) { + if d.aliases[n] { + // TODO this could actually be allowed in some circumstances. + failf("anchor '%s' value contains itself", n.Value) + } + d.aliases[n] = true + d.aliasDepth++ + good = d.unmarshal(n.Alias, out) + d.aliasDepth-- + delete(d.aliases, n) + return good +} + +var zeroValue reflect.Value + +func resetMap(out reflect.Value) { + for _, k := range out.MapKeys() { + out.SetMapIndex(k, zeroValue) + } +} + +func (d *decoder) null(out reflect.Value) bool { + if out.CanAddr() { + switch out.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + out.Set(reflect.Zero(out.Type())) + return true + } + } + return false +} + +func (d *decoder) scalar(n *Node, out reflect.Value) bool { + var tag string + var resolved interface{} + if n.indicatedString() { + tag = strTag + resolved = n.Value + } else { + tag, resolved = resolve(n.Tag, n.Value) + if tag == binaryTag { + data, err := base64.StdEncoding.DecodeString(resolved.(string)) + if err != nil { + failf("!!binary value contains invalid base64 data") + } + resolved = string(data) + } + } + if resolved == nil { + return d.null(out) + } + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + // We've resolved to exactly the type we want, so use that. + out.Set(resolvedv) + return true + } + // Perhaps we can use the value as a TextUnmarshaler to + // set its value. + if out.CanAddr() { + u, ok := out.Addr().Interface().(encoding.TextUnmarshaler) + if ok { + var text []byte + if tag == binaryTag { + text = []byte(resolved.(string)) + } else { + // We let any value be unmarshaled into TextUnmarshaler. + // That might be more lax than we'd like, but the + // TextUnmarshaler itself should bowl out any dubious values. + text = []byte(n.Value) + } + err := u.UnmarshalText(text) + if err != nil { + fail(err) + } + return true + } + } + switch out.Kind() { + case reflect.String: + if tag == binaryTag { + out.SetString(resolved.(string)) + return true + } + out.SetString(n.Value) + return true + case reflect.Interface: + out.Set(reflect.ValueOf(resolved)) + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + // This used to work in v2, but it's very unfriendly. + isDuration := out.Type() == durationType + + switch resolved := resolved.(type) { + case int: + if !isDuration && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case int64: + if !isDuration && !out.OverflowInt(resolved) { + out.SetInt(resolved) + return true + } + case uint64: + if !isDuration && resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case float64: + if !isDuration && resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case string: + if out.Type() == durationType { + d, err := time.ParseDuration(resolved) + if err == nil { + out.SetInt(int64(d)) + return true + } + } + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch resolved := resolved.(type) { + case int: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case int64: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case uint64: + if !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case float64: + if resolved <= math.MaxUint64 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + } + case reflect.Bool: + switch resolved := resolved.(type) { + case bool: + out.SetBool(resolved) + return true + case string: + // This offers some compatibility with the 1.1 spec (https://yaml.org/type/bool.html). + // It only works if explicitly attempting to unmarshal into a typed bool value. + switch resolved { + case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": + out.SetBool(true) + return true + case "n", "N", "no", "No", "NO", "off", "Off", "OFF": + out.SetBool(false) + return true + } + } + case reflect.Float32, reflect.Float64: + switch resolved := resolved.(type) { + case int: + out.SetFloat(float64(resolved)) + return true + case int64: + out.SetFloat(float64(resolved)) + return true + case uint64: + out.SetFloat(float64(resolved)) + return true + case float64: + out.SetFloat(resolved) + return true + } + case reflect.Struct: + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + out.Set(resolvedv) + return true + } + case reflect.Ptr: + panic("yaml internal error: please report the issue") + } + d.terror(n, tag, out) + return false +} + +func settableValueOf(i interface{}) reflect.Value { + v := reflect.ValueOf(i) + sv := reflect.New(v.Type()).Elem() + sv.Set(v) + return sv +} + +func (d *decoder) sequence(n *Node, out reflect.Value) (good bool) { + l := len(n.Content) + + var iface reflect.Value + switch out.Kind() { + case reflect.Slice: + out.Set(reflect.MakeSlice(out.Type(), l, l)) + case reflect.Array: + if l != out.Len() { + failf("invalid array: want %d elements but got %d", out.Len(), l) + } + case reflect.Interface: + // No type hints. Will have to use a generic sequence. + iface = out + out = settableValueOf(make([]interface{}, l)) + default: + d.terror(n, seqTag, out) + return false + } + et := out.Type().Elem() + + j := 0 + for i := 0; i < l; i++ { + e := reflect.New(et).Elem() + if ok := d.unmarshal(n.Content[i], e); ok { + out.Index(j).Set(e) + j++ + } + } + if out.Kind() != reflect.Array { + out.Set(out.Slice(0, j)) + } + if iface.IsValid() { + iface.Set(out) + } + return true +} + +func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) { + l := len(n.Content) + if d.uniqueKeys { + nerrs := len(d.terrors) + for i := 0; i < l; i += 2 { + ni := n.Content[i] + for j := i + 2; j < l; j += 2 { + nj := n.Content[j] + if ni.Kind == nj.Kind && ni.Value == nj.Value { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: mapping key %#v already defined at line %d", nj.Line, nj.Value, ni.Line)) + } + } + } + if len(d.terrors) > nerrs { + return false + } + } + switch out.Kind() { + case reflect.Struct: + return d.mappingStruct(n, out) + case reflect.Map: + // okay + case reflect.Interface: + iface := out + if isStringMap(n) { + out = reflect.MakeMap(d.stringMapType) + } else { + out = reflect.MakeMap(d.generalMapType) + } + iface.Set(out) + default: + d.terror(n, mapTag, out) + return false + } + + outt := out.Type() + kt := outt.Key() + et := outt.Elem() + + stringMapType := d.stringMapType + generalMapType := d.generalMapType + if outt.Elem() == ifaceType { + if outt.Key().Kind() == reflect.String { + d.stringMapType = outt + } else if outt.Key() == ifaceType { + d.generalMapType = outt + } + } + + mergedFields := d.mergedFields + d.mergedFields = nil + + var mergeNode *Node + + mapIsNew := false + if out.IsNil() { + out.Set(reflect.MakeMap(outt)) + mapIsNew = true + } + for i := 0; i < l; i += 2 { + if isMerge(n.Content[i]) { + mergeNode = n.Content[i+1] + continue + } + k := reflect.New(kt).Elem() + if d.unmarshal(n.Content[i], k) { + if mergedFields != nil { + ki := k.Interface() + if mergedFields[ki] { + continue + } + mergedFields[ki] = true + } + kkind := k.Kind() + if kkind == reflect.Interface { + kkind = k.Elem().Kind() + } + if kkind == reflect.Map || kkind == reflect.Slice { + failf("invalid map key: %#v", k.Interface()) + } + e := reflect.New(et).Elem() + if d.unmarshal(n.Content[i+1], e) || n.Content[i+1].ShortTag() == nullTag && (mapIsNew || !out.MapIndex(k).IsValid()) { + out.SetMapIndex(k, e) + } + } + } + + d.mergedFields = mergedFields + if mergeNode != nil { + d.merge(n, mergeNode, out) + } + + d.stringMapType = stringMapType + d.generalMapType = generalMapType + return true +} + +func isStringMap(n *Node) bool { + if n.Kind != MappingNode { + return false + } + l := len(n.Content) + for i := 0; i < l; i += 2 { + shortTag := n.Content[i].ShortTag() + if shortTag != strTag && shortTag != mergeTag { + return false + } + } + return true +} + +func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) { + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + + var inlineMap reflect.Value + var elemType reflect.Type + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + elemType = inlineMap.Type().Elem() + } + + for _, index := range sinfo.InlineUnmarshalers { + field := d.fieldByIndex(n, out, index) + d.prepare(n, field) + } + + mergedFields := d.mergedFields + d.mergedFields = nil + var mergeNode *Node + var doneFields []bool + if d.uniqueKeys { + doneFields = make([]bool, len(sinfo.FieldsList)) + } + name := settableValueOf("") + l := len(n.Content) + for i := 0; i < l; i += 2 { + ni := n.Content[i] + if isMerge(ni) { + mergeNode = n.Content[i+1] + continue + } + if !d.unmarshal(ni, name) { + continue + } + sname := name.String() + if mergedFields != nil { + if mergedFields[sname] { + continue + } + mergedFields[sname] = true + } + if info, ok := sinfo.FieldsMap[sname]; ok { + if d.uniqueKeys { + if doneFields[info.Id] { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.Line, name.String(), out.Type())) + continue + } + doneFields[info.Id] = true + } + var field reflect.Value + if info.Inline == nil { + field = out.Field(info.Num) + } else { + field = d.fieldByIndex(n, out, info.Inline) + } + d.unmarshal(n.Content[i+1], field) + } else if sinfo.InlineMap != -1 { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + value := reflect.New(elemType).Elem() + d.unmarshal(n.Content[i+1], value) + inlineMap.SetMapIndex(name, value) + } else if d.knownFields { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.Line, name.String(), out.Type())) + } + } + + d.mergedFields = mergedFields + if mergeNode != nil { + d.merge(n, mergeNode, out) + } + return true +} + +func failWantMap() { + failf("map merge requires map or sequence of maps as the value") +} + +func (d *decoder) merge(parent *Node, merge *Node, out reflect.Value) { + mergedFields := d.mergedFields + if mergedFields == nil { + d.mergedFields = make(map[interface{}]bool) + for i := 0; i < len(parent.Content); i += 2 { + k := reflect.New(ifaceType).Elem() + if d.unmarshal(parent.Content[i], k) { + d.mergedFields[k.Interface()] = true + } + } + } + + switch merge.Kind { + case MappingNode: + d.unmarshal(merge, out) + case AliasNode: + if merge.Alias != nil && merge.Alias.Kind != MappingNode { + failWantMap() + } + d.unmarshal(merge, out) + case SequenceNode: + for i := 0; i < len(merge.Content); i++ { + ni := merge.Content[i] + if ni.Kind == AliasNode { + if ni.Alias != nil && ni.Alias.Kind != MappingNode { + failWantMap() + } + } else if ni.Kind != MappingNode { + failWantMap() + } + d.unmarshal(ni, out) + } + default: + failWantMap() + } + + d.mergedFields = mergedFields +} + +func isMerge(n *Node) bool { + return n.Kind == ScalarNode && n.Value == "<<" && (n.Tag == "" || n.Tag == "!" || shortTag(n.Tag) == mergeTag) +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/emitterc.go b/scraper-go/vendor/gopkg.in/yaml.v3/emitterc.go new file mode 100644 index 00000000..0f47c9ca --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/emitterc.go @@ -0,0 +1,2020 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "bytes" + "fmt" +) + +// Flush the buffer if needed. +func flush(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) { + return yaml_emitter_flush(emitter) + } + return true +} + +// Put a character to the output buffer. +func put(emitter *yaml_emitter_t, value byte) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + emitter.buffer[emitter.buffer_pos] = value + emitter.buffer_pos++ + emitter.column++ + return true +} + +// Put a line break to the output buffer. +func put_break(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + switch emitter.line_break { + case yaml_CR_BREAK: + emitter.buffer[emitter.buffer_pos] = '\r' + emitter.buffer_pos += 1 + case yaml_LN_BREAK: + emitter.buffer[emitter.buffer_pos] = '\n' + emitter.buffer_pos += 1 + case yaml_CRLN_BREAK: + emitter.buffer[emitter.buffer_pos+0] = '\r' + emitter.buffer[emitter.buffer_pos+1] = '\n' + emitter.buffer_pos += 2 + default: + panic("unknown line break setting") + } + if emitter.column == 0 { + emitter.space_above = true + } + emitter.column = 0 + emitter.line++ + // [Go] Do this here and below and drop from everywhere else (see commented lines). + emitter.indention = true + return true +} + +// Copy a character from a string into buffer. +func write(emitter *yaml_emitter_t, s []byte, i *int) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + p := emitter.buffer_pos + w := width(s[*i]) + switch w { + case 4: + emitter.buffer[p+3] = s[*i+3] + fallthrough + case 3: + emitter.buffer[p+2] = s[*i+2] + fallthrough + case 2: + emitter.buffer[p+1] = s[*i+1] + fallthrough + case 1: + emitter.buffer[p+0] = s[*i+0] + default: + panic("unknown character width") + } + emitter.column++ + emitter.buffer_pos += w + *i += w + return true +} + +// Write a whole string into buffer. +func write_all(emitter *yaml_emitter_t, s []byte) bool { + for i := 0; i < len(s); { + if !write(emitter, s, &i) { + return false + } + } + return true +} + +// Copy a line break character from a string into buffer. +func write_break(emitter *yaml_emitter_t, s []byte, i *int) bool { + if s[*i] == '\n' { + if !put_break(emitter) { + return false + } + *i++ + } else { + if !write(emitter, s, i) { + return false + } + if emitter.column == 0 { + emitter.space_above = true + } + emitter.column = 0 + emitter.line++ + // [Go] Do this here and above and drop from everywhere else (see commented lines). + emitter.indention = true + } + return true +} + +// Set an emitter error and return false. +func yaml_emitter_set_emitter_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_EMITTER_ERROR + emitter.problem = problem + return false +} + +// Emit an event. +func yaml_emitter_emit(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.events = append(emitter.events, *event) + for !yaml_emitter_need_more_events(emitter) { + event := &emitter.events[emitter.events_head] + if !yaml_emitter_analyze_event(emitter, event) { + return false + } + if !yaml_emitter_state_machine(emitter, event) { + return false + } + yaml_event_delete(event) + emitter.events_head++ + } + return true +} + +// Check if we need to accumulate more events before emitting. +// +// We accumulate extra +// - 1 event for DOCUMENT-START +// - 2 events for SEQUENCE-START +// - 3 events for MAPPING-START +// +func yaml_emitter_need_more_events(emitter *yaml_emitter_t) bool { + if emitter.events_head == len(emitter.events) { + return true + } + var accumulate int + switch emitter.events[emitter.events_head].typ { + case yaml_DOCUMENT_START_EVENT: + accumulate = 1 + break + case yaml_SEQUENCE_START_EVENT: + accumulate = 2 + break + case yaml_MAPPING_START_EVENT: + accumulate = 3 + break + default: + return false + } + if len(emitter.events)-emitter.events_head > accumulate { + return false + } + var level int + for i := emitter.events_head; i < len(emitter.events); i++ { + switch emitter.events[i].typ { + case yaml_STREAM_START_EVENT, yaml_DOCUMENT_START_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT: + level++ + case yaml_STREAM_END_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_END_EVENT, yaml_MAPPING_END_EVENT: + level-- + } + if level == 0 { + return false + } + } + return true +} + +// Append a directive to the directives stack. +func yaml_emitter_append_tag_directive(emitter *yaml_emitter_t, value *yaml_tag_directive_t, allow_duplicates bool) bool { + for i := 0; i < len(emitter.tag_directives); i++ { + if bytes.Equal(value.handle, emitter.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_emitter_set_emitter_error(emitter, "duplicate %TAG directive") + } + } + + // [Go] Do we actually need to copy this given garbage collection + // and the lack of deallocating destructors? + tag_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(tag_copy.handle, value.handle) + copy(tag_copy.prefix, value.prefix) + emitter.tag_directives = append(emitter.tag_directives, tag_copy) + return true +} + +// Increase the indentation level. +func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool) bool { + emitter.indents = append(emitter.indents, emitter.indent) + if emitter.indent < 0 { + if flow { + emitter.indent = emitter.best_indent + } else { + emitter.indent = 0 + } + } else if !indentless { + // [Go] This was changed so that indentations are more regular. + if emitter.states[len(emitter.states)-1] == yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE { + // The first indent inside a sequence will just skip the "- " indicator. + emitter.indent += 2 + } else { + // Everything else aligns to the chosen indentation. + emitter.indent = emitter.best_indent*((emitter.indent+emitter.best_indent)/emitter.best_indent) + } + } + return true +} + +// State dispatcher. +func yaml_emitter_state_machine(emitter *yaml_emitter_t, event *yaml_event_t) bool { + switch emitter.state { + default: + case yaml_EMIT_STREAM_START_STATE: + return yaml_emitter_emit_stream_start(emitter, event) + + case yaml_EMIT_FIRST_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, true) + + case yaml_EMIT_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, false) + + case yaml_EMIT_DOCUMENT_CONTENT_STATE: + return yaml_emitter_emit_document_content(emitter, event) + + case yaml_EMIT_DOCUMENT_END_STATE: + return yaml_emitter_emit_document_end(emitter, event) + + case yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, true, false) + + case yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false, true) + + case yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false, false) + + case yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, true, false) + + case yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false, true) + + case yaml_EMIT_FLOW_MAPPING_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false, false) + + case yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, false) + + case yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, true) + + case yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, false) + + case yaml_EMIT_END_STATE: + return yaml_emitter_set_emitter_error(emitter, "expected nothing after STREAM-END") + } + panic("invalid emitter state") +} + +// Expect STREAM-START. +func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_STREAM_START_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected STREAM-START") + } + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = event.encoding + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = yaml_UTF8_ENCODING + } + } + if emitter.best_indent < 2 || emitter.best_indent > 9 { + emitter.best_indent = 2 + } + if emitter.best_width >= 0 && emitter.best_width <= emitter.best_indent*2 { + emitter.best_width = 80 + } + if emitter.best_width < 0 { + emitter.best_width = 1<<31 - 1 + } + if emitter.line_break == yaml_ANY_BREAK { + emitter.line_break = yaml_LN_BREAK + } + + emitter.indent = -1 + emitter.line = 0 + emitter.column = 0 + emitter.whitespace = true + emitter.indention = true + emitter.space_above = true + emitter.foot_indent = -1 + + if emitter.encoding != yaml_UTF8_ENCODING { + if !yaml_emitter_write_bom(emitter) { + return false + } + } + emitter.state = yaml_EMIT_FIRST_DOCUMENT_START_STATE + return true +} + +// Expect DOCUMENT-START or STREAM-END. +func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + + if event.typ == yaml_DOCUMENT_START_EVENT { + + if event.version_directive != nil { + if !yaml_emitter_analyze_version_directive(emitter, event.version_directive) { + return false + } + } + + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_analyze_tag_directive(emitter, tag_directive) { + return false + } + if !yaml_emitter_append_tag_directive(emitter, tag_directive, false) { + return false + } + } + + for i := 0; i < len(default_tag_directives); i++ { + tag_directive := &default_tag_directives[i] + if !yaml_emitter_append_tag_directive(emitter, tag_directive, true) { + return false + } + } + + implicit := event.implicit + if !first || emitter.canonical { + implicit = false + } + + if emitter.open_ended && (event.version_directive != nil || len(event.tag_directives) > 0) { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if event.version_directive != nil { + implicit = false + if !yaml_emitter_write_indicator(emitter, []byte("%YAML"), true, false, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("1.1"), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if len(event.tag_directives) > 0 { + implicit = false + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_write_indicator(emitter, []byte("%TAG"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_handle(emitter, tag_directive.handle) { + return false + } + if !yaml_emitter_write_tag_content(emitter, tag_directive.prefix, true) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if yaml_emitter_check_empty_document(emitter) { + implicit = false + } + if !implicit { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("---"), true, false, false) { + return false + } + if emitter.canonical || true { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if len(emitter.head_comment) > 0 { + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if !put_break(emitter) { + return false + } + } + + emitter.state = yaml_EMIT_DOCUMENT_CONTENT_STATE + return true + } + + if event.typ == yaml_STREAM_END_EVENT { + if emitter.open_ended { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_END_STATE + return true + } + + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-START or STREAM-END") +} + +// Expect the root node. +func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.states = append(emitter.states, yaml_EMIT_DOCUMENT_END_STATE) + + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if !yaml_emitter_emit_node(emitter, event, true, false, false, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect DOCUMENT-END. +func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_DOCUMENT_END_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-END") + } + // [Go] Force document foot separation. + emitter.foot_indent = 0 + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + emitter.foot_indent = -1 + if !yaml_emitter_write_indent(emitter) { + return false + } + if !event.implicit { + // [Go] Allocate the slice elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_DOCUMENT_START_STATE + emitter.tag_directives = emitter.tag_directives[:0] + return true +} + +// Expect a flow item node. +func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first, trail bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_SEQUENCE_END_EVENT { + if emitter.canonical && !first && !trail { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.column == 0 || emitter.canonical && !first { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + + return true + } + + if !first && !trail { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if emitter.column == 0 { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if len(emitter.line_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0 { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE) + } else { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE) + } + if !yaml_emitter_emit_node(emitter, event, false, true, false, false) { + return false + } + if len(emitter.line_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0 { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect a flow key node. +func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first, trail bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_MAPPING_END_EVENT { + if (emitter.canonical || len(emitter.head_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0) && !first && !trail { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if !yaml_emitter_process_head_comment(emitter) { + return false + } + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + + if !first && !trail { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if !yaml_emitter_process_head_comment(emitter) { + return false + } + + if emitter.column == 0 { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if !emitter.canonical && yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, false) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a flow value node. +func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, false) { + return false + } + } + if len(emitter.line_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0 { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE) + } else { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE) + } + if !yaml_emitter_emit_node(emitter, event, false, false, true, false) { + return false + } + if len(emitter.line_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0 { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect a block item node. +func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, false) { + return false + } + } + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'-'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE) + if !yaml_emitter_emit_node(emitter, event, false, true, false, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect a block key node. +func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, false) { + return false + } + } + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if event.typ == yaml_MAPPING_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if len(emitter.line_comment) > 0 { + // [Go] A line comment was provided for the key. That's unusual as the + // scanner associates line comments with the value. Either way, + // save the line comment and render it appropriately later. + emitter.key_line_comment = emitter.line_comment + emitter.line_comment = nil + } + if yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block value node. +func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, true) { + return false + } + } + if len(emitter.key_line_comment) > 0 { + // [Go] Line comments are generally associated with the value, but when there's + // no value on the same line as a mapping key they end up attached to the + // key itself. + if event.typ == yaml_SCALAR_EVENT { + if len(emitter.line_comment) == 0 { + // A scalar is coming and it has no line comments by itself yet, + // so just let it handle the line comment as usual. If it has a + // line comment, we can't have both so the one from the key is lost. + emitter.line_comment = emitter.key_line_comment + emitter.key_line_comment = nil + } + } else if event.sequence_style() != yaml_FLOW_SEQUENCE_STYLE && (event.typ == yaml_MAPPING_START_EVENT || event.typ == yaml_SEQUENCE_START_EVENT) { + // An indented block follows, so write the comment right now. + emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment + if !yaml_emitter_process_line_comment(emitter) { + return false + } + emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment + } + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) + if !yaml_emitter_emit_node(emitter, event, false, false, true, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +func yaml_emitter_silent_nil_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + return event.typ == yaml_SCALAR_EVENT && event.implicit && !emitter.canonical && len(emitter.scalar_data.value) == 0 +} + +// Expect a node. +func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, + root bool, sequence bool, mapping bool, simple_key bool) bool { + + emitter.root_context = root + emitter.sequence_context = sequence + emitter.mapping_context = mapping + emitter.simple_key_context = simple_key + + switch event.typ { + case yaml_ALIAS_EVENT: + return yaml_emitter_emit_alias(emitter, event) + case yaml_SCALAR_EVENT: + return yaml_emitter_emit_scalar(emitter, event) + case yaml_SEQUENCE_START_EVENT: + return yaml_emitter_emit_sequence_start(emitter, event) + case yaml_MAPPING_START_EVENT: + return yaml_emitter_emit_mapping_start(emitter, event) + default: + return yaml_emitter_set_emitter_error(emitter, + fmt.Sprintf("expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS, but got %v", event.typ)) + } +} + +// Expect ALIAS. +func yaml_emitter_emit_alias(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SCALAR. +func yaml_emitter_emit_scalar(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_select_scalar_style(emitter, event) { + return false + } + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + if !yaml_emitter_process_scalar(emitter) { + return false + } + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SEQUENCE-START. +func yaml_emitter_emit_sequence_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || + yaml_emitter_check_empty_sequence(emitter) { + emitter.state = yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE + } + return true +} + +// Expect MAPPING-START. +func yaml_emitter_emit_mapping_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.mapping_style() == yaml_FLOW_MAPPING_STYLE || + yaml_emitter_check_empty_mapping(emitter) { + emitter.state = yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE + } + return true +} + +// Check if the document content is an empty scalar. +func yaml_emitter_check_empty_document(emitter *yaml_emitter_t) bool { + return false // [Go] Huh? +} + +// Check if the next events represent an empty sequence. +func yaml_emitter_check_empty_sequence(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_SEQUENCE_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_SEQUENCE_END_EVENT +} + +// Check if the next events represent an empty mapping. +func yaml_emitter_check_empty_mapping(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_MAPPING_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_MAPPING_END_EVENT +} + +// Check if the next node can be expressed as a simple key. +func yaml_emitter_check_simple_key(emitter *yaml_emitter_t) bool { + length := 0 + switch emitter.events[emitter.events_head].typ { + case yaml_ALIAS_EVENT: + length += len(emitter.anchor_data.anchor) + case yaml_SCALAR_EVENT: + if emitter.scalar_data.multiline { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + + len(emitter.scalar_data.value) + case yaml_SEQUENCE_START_EVENT: + if !yaml_emitter_check_empty_sequence(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + case yaml_MAPPING_START_EVENT: + if !yaml_emitter_check_empty_mapping(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + default: + return false + } + return length <= 128 +} + +// Determine an acceptable scalar style. +func yaml_emitter_select_scalar_style(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + no_tag := len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 + if no_tag && !event.implicit && !event.quoted_implicit { + return yaml_emitter_set_emitter_error(emitter, "neither tag nor implicit flags are specified") + } + + style := event.scalar_style() + if style == yaml_ANY_SCALAR_STYLE { + style = yaml_PLAIN_SCALAR_STYLE + } + if emitter.canonical { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + if emitter.simple_key_context && emitter.scalar_data.multiline { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + if style == yaml_PLAIN_SCALAR_STYLE { + if emitter.flow_level > 0 && !emitter.scalar_data.flow_plain_allowed || + emitter.flow_level == 0 && !emitter.scalar_data.block_plain_allowed { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if len(emitter.scalar_data.value) == 0 && (emitter.flow_level > 0 || emitter.simple_key_context) { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if no_tag && !event.implicit { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_SINGLE_QUOTED_SCALAR_STYLE { + if !emitter.scalar_data.single_quoted_allowed { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_LITERAL_SCALAR_STYLE || style == yaml_FOLDED_SCALAR_STYLE { + if !emitter.scalar_data.block_allowed || emitter.flow_level > 0 || emitter.simple_key_context { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + + if no_tag && !event.quoted_implicit && style != yaml_PLAIN_SCALAR_STYLE { + emitter.tag_data.handle = []byte{'!'} + } + emitter.scalar_data.style = style + return true +} + +// Write an anchor. +func yaml_emitter_process_anchor(emitter *yaml_emitter_t) bool { + if emitter.anchor_data.anchor == nil { + return true + } + c := []byte{'&'} + if emitter.anchor_data.alias { + c[0] = '*' + } + if !yaml_emitter_write_indicator(emitter, c, true, false, false) { + return false + } + return yaml_emitter_write_anchor(emitter, emitter.anchor_data.anchor) +} + +// Write a tag. +func yaml_emitter_process_tag(emitter *yaml_emitter_t) bool { + if len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 { + return true + } + if len(emitter.tag_data.handle) > 0 { + if !yaml_emitter_write_tag_handle(emitter, emitter.tag_data.handle) { + return false + } + if len(emitter.tag_data.suffix) > 0 { + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + } + } else { + // [Go] Allocate these slices elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("!<"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, false, false, false) { + return false + } + } + return true +} + +// Write a scalar. +func yaml_emitter_process_scalar(emitter *yaml_emitter_t) bool { + switch emitter.scalar_data.style { + case yaml_PLAIN_SCALAR_STYLE: + return yaml_emitter_write_plain_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_SINGLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_single_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_DOUBLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_double_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_LITERAL_SCALAR_STYLE: + return yaml_emitter_write_literal_scalar(emitter, emitter.scalar_data.value) + + case yaml_FOLDED_SCALAR_STYLE: + return yaml_emitter_write_folded_scalar(emitter, emitter.scalar_data.value) + } + panic("unknown scalar style") +} + +// Write a head comment. +func yaml_emitter_process_head_comment(emitter *yaml_emitter_t) bool { + if len(emitter.tail_comment) > 0 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_comment(emitter, emitter.tail_comment) { + return false + } + emitter.tail_comment = emitter.tail_comment[:0] + emitter.foot_indent = emitter.indent + if emitter.foot_indent < 0 { + emitter.foot_indent = 0 + } + } + + if len(emitter.head_comment) == 0 { + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_comment(emitter, emitter.head_comment) { + return false + } + emitter.head_comment = emitter.head_comment[:0] + return true +} + +// Write an line comment. +func yaml_emitter_process_line_comment(emitter *yaml_emitter_t) bool { + if len(emitter.line_comment) == 0 { + return true + } + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !yaml_emitter_write_comment(emitter, emitter.line_comment) { + return false + } + emitter.line_comment = emitter.line_comment[:0] + return true +} + +// Write a foot comment. +func yaml_emitter_process_foot_comment(emitter *yaml_emitter_t) bool { + if len(emitter.foot_comment) == 0 { + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_comment(emitter, emitter.foot_comment) { + return false + } + emitter.foot_comment = emitter.foot_comment[:0] + emitter.foot_indent = emitter.indent + if emitter.foot_indent < 0 { + emitter.foot_indent = 0 + } + return true +} + +// Check if a %YAML directive is valid. +func yaml_emitter_analyze_version_directive(emitter *yaml_emitter_t, version_directive *yaml_version_directive_t) bool { + if version_directive.major != 1 || version_directive.minor != 1 { + return yaml_emitter_set_emitter_error(emitter, "incompatible %YAML directive") + } + return true +} + +// Check if a %TAG directive is valid. +func yaml_emitter_analyze_tag_directive(emitter *yaml_emitter_t, tag_directive *yaml_tag_directive_t) bool { + handle := tag_directive.handle + prefix := tag_directive.prefix + if len(handle) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag handle must not be empty") + } + if handle[0] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must start with '!'") + } + if handle[len(handle)-1] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must end with '!'") + } + for i := 1; i < len(handle)-1; i += width(handle[i]) { + if !is_alpha(handle, i) { + return yaml_emitter_set_emitter_error(emitter, "tag handle must contain alphanumerical characters only") + } + } + if len(prefix) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag prefix must not be empty") + } + return true +} + +// Check if an anchor is valid. +func yaml_emitter_analyze_anchor(emitter *yaml_emitter_t, anchor []byte, alias bool) bool { + if len(anchor) == 0 { + problem := "anchor value must not be empty" + if alias { + problem = "alias value must not be empty" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + for i := 0; i < len(anchor); i += width(anchor[i]) { + if !is_alpha(anchor, i) { + problem := "anchor value must contain alphanumerical characters only" + if alias { + problem = "alias value must contain alphanumerical characters only" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + } + emitter.anchor_data.anchor = anchor + emitter.anchor_data.alias = alias + return true +} + +// Check if a tag is valid. +func yaml_emitter_analyze_tag(emitter *yaml_emitter_t, tag []byte) bool { + if len(tag) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag value must not be empty") + } + for i := 0; i < len(emitter.tag_directives); i++ { + tag_directive := &emitter.tag_directives[i] + if bytes.HasPrefix(tag, tag_directive.prefix) { + emitter.tag_data.handle = tag_directive.handle + emitter.tag_data.suffix = tag[len(tag_directive.prefix):] + return true + } + } + emitter.tag_data.suffix = tag + return true +} + +// Check if a scalar is valid. +func yaml_emitter_analyze_scalar(emitter *yaml_emitter_t, value []byte) bool { + var ( + block_indicators = false + flow_indicators = false + line_breaks = false + special_characters = false + tab_characters = false + + leading_space = false + leading_break = false + trailing_space = false + trailing_break = false + break_space = false + space_break = false + + preceded_by_whitespace = false + followed_by_whitespace = false + previous_space = false + previous_break = false + ) + + emitter.scalar_data.value = value + + if len(value) == 0 { + emitter.scalar_data.multiline = false + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = false + return true + } + + if len(value) >= 3 && ((value[0] == '-' && value[1] == '-' && value[2] == '-') || (value[0] == '.' && value[1] == '.' && value[2] == '.')) { + block_indicators = true + flow_indicators = true + } + + preceded_by_whitespace = true + for i, w := 0, 0; i < len(value); i += w { + w = width(value[i]) + followed_by_whitespace = i+w >= len(value) || is_blank(value, i+w) + + if i == 0 { + switch value[i] { + case '#', ',', '[', ']', '{', '}', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`': + flow_indicators = true + block_indicators = true + case '?', ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '-': + if followed_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } else { + switch value[i] { + case ',', '?', '[', ']', '{', '}': + flow_indicators = true + case ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '#': + if preceded_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } + + if value[i] == '\t' { + tab_characters = true + } else if !is_printable(value, i) || !is_ascii(value, i) && !emitter.unicode { + special_characters = true + } + if is_space(value, i) { + if i == 0 { + leading_space = true + } + if i+width(value[i]) == len(value) { + trailing_space = true + } + if previous_break { + break_space = true + } + previous_space = true + previous_break = false + } else if is_break(value, i) { + line_breaks = true + if i == 0 { + leading_break = true + } + if i+width(value[i]) == len(value) { + trailing_break = true + } + if previous_space { + space_break = true + } + previous_space = false + previous_break = true + } else { + previous_space = false + previous_break = false + } + + // [Go]: Why 'z'? Couldn't be the end of the string as that's the loop condition. + preceded_by_whitespace = is_blankz(value, i) + } + + emitter.scalar_data.multiline = line_breaks + emitter.scalar_data.flow_plain_allowed = true + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = true + + if leading_space || leading_break || trailing_space || trailing_break { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if trailing_space { + emitter.scalar_data.block_allowed = false + } + if break_space { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || tab_characters || special_characters { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || special_characters { + emitter.scalar_data.block_allowed = false + } + if line_breaks { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if flow_indicators { + emitter.scalar_data.flow_plain_allowed = false + } + if block_indicators { + emitter.scalar_data.block_plain_allowed = false + } + return true +} + +// Check if the event data is valid. +func yaml_emitter_analyze_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + emitter.anchor_data.anchor = nil + emitter.tag_data.handle = nil + emitter.tag_data.suffix = nil + emitter.scalar_data.value = nil + + if len(event.head_comment) > 0 { + emitter.head_comment = event.head_comment + } + if len(event.line_comment) > 0 { + emitter.line_comment = event.line_comment + } + if len(event.foot_comment) > 0 { + emitter.foot_comment = event.foot_comment + } + if len(event.tail_comment) > 0 { + emitter.tail_comment = event.tail_comment + } + + switch event.typ { + case yaml_ALIAS_EVENT: + if !yaml_emitter_analyze_anchor(emitter, event.anchor, true) { + return false + } + + case yaml_SCALAR_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || (!event.implicit && !event.quoted_implicit)) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + if !yaml_emitter_analyze_scalar(emitter, event.value) { + return false + } + + case yaml_SEQUENCE_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + + case yaml_MAPPING_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + } + return true +} + +// Write the BOM character. +func yaml_emitter_write_bom(emitter *yaml_emitter_t) bool { + if !flush(emitter) { + return false + } + pos := emitter.buffer_pos + emitter.buffer[pos+0] = '\xEF' + emitter.buffer[pos+1] = '\xBB' + emitter.buffer[pos+2] = '\xBF' + emitter.buffer_pos += 3 + return true +} + +func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool { + indent := emitter.indent + if indent < 0 { + indent = 0 + } + if !emitter.indention || emitter.column > indent || (emitter.column == indent && !emitter.whitespace) { + if !put_break(emitter) { + return false + } + } + if emitter.foot_indent == indent { + if !put_break(emitter) { + return false + } + } + for emitter.column < indent { + if !put(emitter, ' ') { + return false + } + } + emitter.whitespace = true + //emitter.indention = true + emitter.space_above = false + emitter.foot_indent = -1 + return true +} + +func yaml_emitter_write_indicator(emitter *yaml_emitter_t, indicator []byte, need_whitespace, is_whitespace, is_indention bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, indicator) { + return false + } + emitter.whitespace = is_whitespace + emitter.indention = (emitter.indention && is_indention) + emitter.open_ended = false + return true +} + +func yaml_emitter_write_anchor(emitter *yaml_emitter_t, value []byte) bool { + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_handle(emitter *yaml_emitter_t, value []byte) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_whitespace bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + for i := 0; i < len(value); { + var must_write bool + switch value[i] { + case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')', '[', ']': + must_write = true + default: + must_write = is_alpha(value, i) + } + if must_write { + if !write(emitter, value, &i) { + return false + } + } else { + w := width(value[i]) + for k := 0; k < w; k++ { + octet := value[i] + i++ + if !put(emitter, '%') { + return false + } + + c := octet >> 4 + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + + c = octet & 0x0f + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + } + } + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + if len(value) > 0 && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + //emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + + if len(value) > 0 { + emitter.whitespace = false + } + emitter.indention = false + if emitter.root_context { + emitter.open_ended = true + } + + return true +} + +func yaml_emitter_write_single_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, true, false, false) { + return false + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + //emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if value[i] == '\'' { + if !put(emitter, '\'') { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_double_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + spaces := false + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, true, false, false) { + return false + } + + for i := 0; i < len(value); { + if !is_printable(value, i) || (!emitter.unicode && !is_ascii(value, i)) || + is_bom(value, i) || is_break(value, i) || + value[i] == '"' || value[i] == '\\' { + + octet := value[i] + + var w int + var v rune + switch { + case octet&0x80 == 0x00: + w, v = 1, rune(octet&0x7F) + case octet&0xE0 == 0xC0: + w, v = 2, rune(octet&0x1F) + case octet&0xF0 == 0xE0: + w, v = 3, rune(octet&0x0F) + case octet&0xF8 == 0xF0: + w, v = 4, rune(octet&0x07) + } + for k := 1; k < w; k++ { + octet = value[i+k] + v = (v << 6) + (rune(octet) & 0x3F) + } + i += w + + if !put(emitter, '\\') { + return false + } + + var ok bool + switch v { + case 0x00: + ok = put(emitter, '0') + case 0x07: + ok = put(emitter, 'a') + case 0x08: + ok = put(emitter, 'b') + case 0x09: + ok = put(emitter, 't') + case 0x0A: + ok = put(emitter, 'n') + case 0x0b: + ok = put(emitter, 'v') + case 0x0c: + ok = put(emitter, 'f') + case 0x0d: + ok = put(emitter, 'r') + case 0x1b: + ok = put(emitter, 'e') + case 0x22: + ok = put(emitter, '"') + case 0x5c: + ok = put(emitter, '\\') + case 0x85: + ok = put(emitter, 'N') + case 0xA0: + ok = put(emitter, '_') + case 0x2028: + ok = put(emitter, 'L') + case 0x2029: + ok = put(emitter, 'P') + default: + if v <= 0xFF { + ok = put(emitter, 'x') + w = 2 + } else if v <= 0xFFFF { + ok = put(emitter, 'u') + w = 4 + } else { + ok = put(emitter, 'U') + w = 8 + } + for k := (w - 1) * 4; ok && k >= 0; k -= 4 { + digit := byte((v >> uint(k)) & 0x0F) + if digit < 10 { + ok = put(emitter, digit+'0') + } else { + ok = put(emitter, digit+'A'-10) + } + } + } + if !ok { + return false + } + spaces = false + } else if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if is_space(value, i+1) { + if !put(emitter, '\\') { + return false + } + } + i += width(value[i]) + } else if !write(emitter, value, &i) { + return false + } + spaces = true + } else { + if !write(emitter, value, &i) { + return false + } + spaces = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_block_scalar_hints(emitter *yaml_emitter_t, value []byte) bool { + if is_space(value, 0) || is_break(value, 0) { + indent_hint := []byte{'0' + byte(emitter.best_indent)} + if !yaml_emitter_write_indicator(emitter, indent_hint, false, false, false) { + return false + } + } + + emitter.open_ended = false + + var chomp_hint [1]byte + if len(value) == 0 { + chomp_hint[0] = '-' + } else { + i := len(value) - 1 + for value[i]&0xC0 == 0x80 { + i-- + } + if !is_break(value, i) { + chomp_hint[0] = '-' + } else if i == 0 { + chomp_hint[0] = '+' + emitter.open_ended = true + } else { + i-- + for value[i]&0xC0 == 0x80 { + i-- + } + if is_break(value, i) { + chomp_hint[0] = '+' + emitter.open_ended = true + } + } + } + if chomp_hint[0] != 0 { + if !yaml_emitter_write_indicator(emitter, chomp_hint[:], false, false, false) { + return false + } + } + return true +} + +func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'|'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + //emitter.indention = true + emitter.whitespace = true + breaks := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !write_break(emitter, value, &i) { + return false + } + //emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + + return true +} + +func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + + //emitter.indention = true + emitter.whitespace = true + + breaks := true + leading_spaces := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !breaks && !leading_spaces && value[i] == '\n' { + k := 0 + for is_break(value, k) { + k += width(value[k]) + } + if !is_blankz(value, k) { + if !put_break(emitter) { + return false + } + } + } + if !write_break(emitter, value, &i) { + return false + } + //emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + leading_spaces = is_blank(value, i) + } + if !breaks && is_space(value, i) && !is_space(value, i+1) && emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + emitter.indention = false + breaks = false + } + } + return true +} + +func yaml_emitter_write_comment(emitter *yaml_emitter_t, comment []byte) bool { + breaks := false + pound := false + for i := 0; i < len(comment); { + if is_break(comment, i) { + if !write_break(emitter, comment, &i) { + return false + } + //emitter.indention = true + breaks = true + pound = false + } else { + if breaks && !yaml_emitter_write_indent(emitter) { + return false + } + if !pound { + if comment[i] != '#' && (!put(emitter, '#') || !put(emitter, ' ')) { + return false + } + pound = true + } + if !write(emitter, comment, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + if !breaks && !put_break(emitter) { + return false + } + + emitter.whitespace = true + //emitter.indention = true + return true +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/encode.go b/scraper-go/vendor/gopkg.in/yaml.v3/encode.go new file mode 100644 index 00000000..de9e72a3 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/encode.go @@ -0,0 +1,577 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "encoding" + "fmt" + "io" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type encoder struct { + emitter yaml_emitter_t + event yaml_event_t + out []byte + flow bool + indent int + doneInit bool +} + +func newEncoder() *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_string(&e.emitter, &e.out) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func newEncoderWithWriter(w io.Writer) *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_writer(&e.emitter, w) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func (e *encoder) init() { + if e.doneInit { + return + } + if e.indent == 0 { + e.indent = 4 + } + e.emitter.best_indent = e.indent + yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING) + e.emit() + e.doneInit = true +} + +func (e *encoder) finish() { + e.emitter.open_ended = false + yaml_stream_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) destroy() { + yaml_emitter_delete(&e.emitter) +} + +func (e *encoder) emit() { + // This will internally delete the e.event value. + e.must(yaml_emitter_emit(&e.emitter, &e.event)) +} + +func (e *encoder) must(ok bool) { + if !ok { + msg := e.emitter.problem + if msg == "" { + msg = "unknown problem generating YAML content" + } + failf("%s", msg) + } +} + +func (e *encoder) marshalDoc(tag string, in reflect.Value) { + e.init() + var node *Node + if in.IsValid() { + node, _ = in.Interface().(*Node) + } + if node != nil && node.Kind == DocumentNode { + e.nodev(in) + } else { + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.emit() + e.marshal(tag, in) + yaml_document_end_event_initialize(&e.event, true) + e.emit() + } +} + +func (e *encoder) marshal(tag string, in reflect.Value) { + tag = shortTag(tag) + if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() { + e.nilv() + return + } + iface := in.Interface() + switch value := iface.(type) { + case *Node: + e.nodev(in) + return + case Node: + if !in.CanAddr() { + var n = reflect.New(in.Type()).Elem() + n.Set(in) + in = n + } + e.nodev(in.Addr()) + return + case time.Time: + e.timev(tag, in) + return + case *time.Time: + e.timev(tag, in.Elem()) + return + case time.Duration: + e.stringv(tag, reflect.ValueOf(value.String())) + return + case Marshaler: + v, err := value.MarshalYAML() + if err != nil { + fail(err) + } + if v == nil { + e.nilv() + return + } + e.marshal(tag, reflect.ValueOf(v)) + return + case encoding.TextMarshaler: + text, err := value.MarshalText() + if err != nil { + fail(err) + } + in = reflect.ValueOf(string(text)) + case nil: + e.nilv() + return + } + switch in.Kind() { + case reflect.Interface: + e.marshal(tag, in.Elem()) + case reflect.Map: + e.mapv(tag, in) + case reflect.Ptr: + e.marshal(tag, in.Elem()) + case reflect.Struct: + e.structv(tag, in) + case reflect.Slice, reflect.Array: + e.slicev(tag, in) + case reflect.String: + e.stringv(tag, in) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + e.intv(tag, in) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + e.uintv(tag, in) + case reflect.Float32, reflect.Float64: + e.floatv(tag, in) + case reflect.Bool: + e.boolv(tag, in) + default: + panic("cannot marshal type: " + in.Type().String()) + } +} + +func (e *encoder) mapv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + keys := keyList(in.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + e.marshal("", k) + e.marshal("", in.MapIndex(k)) + } + }) +} + +func (e *encoder) fieldByIndex(v reflect.Value, index []int) (field reflect.Value) { + for _, num := range index { + for { + if v.Kind() == reflect.Ptr { + if v.IsNil() { + return reflect.Value{} + } + v = v.Elem() + continue + } + break + } + v = v.Field(num) + } + return v +} + +func (e *encoder) structv(tag string, in reflect.Value) { + sinfo, err := getStructInfo(in.Type()) + if err != nil { + panic(err) + } + e.mappingv(tag, func() { + for _, info := range sinfo.FieldsList { + var value reflect.Value + if info.Inline == nil { + value = in.Field(info.Num) + } else { + value = e.fieldByIndex(in, info.Inline) + if !value.IsValid() { + continue + } + } + if info.OmitEmpty && isZero(value) { + continue + } + e.marshal("", reflect.ValueOf(info.Key)) + e.flow = info.Flow + e.marshal("", value) + } + if sinfo.InlineMap >= 0 { + m := in.Field(sinfo.InlineMap) + if m.Len() > 0 { + e.flow = false + keys := keyList(m.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + if _, found := sinfo.FieldsMap[k.String()]; found { + panic(fmt.Sprintf("cannot have key %q in inlined map: conflicts with struct field", k.String())) + } + e.marshal("", k) + e.flow = false + e.marshal("", m.MapIndex(k)) + } + } + } + }) +} + +func (e *encoder) mappingv(tag string, f func()) { + implicit := tag == "" + style := yaml_BLOCK_MAPPING_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, nil, []byte(tag), implicit, style) + e.emit() + f() + yaml_mapping_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) slicev(tag string, in reflect.Value) { + implicit := tag == "" + style := yaml_BLOCK_SEQUENCE_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, nil, []byte(tag), implicit, style)) + e.emit() + n := in.Len() + for i := 0; i < n; i++ { + e.marshal("", in.Index(i)) + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.emit() +} + +// isBase60 returns whether s is in base 60 notation as defined in YAML 1.1. +// +// The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported +// in YAML 1.2 and by this package, but these should be marshalled quoted for +// the time being for compatibility with other parsers. +func isBase60Float(s string) (result bool) { + // Fast path. + if s == "" { + return false + } + c := s[0] + if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 { + return false + } + // Do the full match. + return base60float.MatchString(s) +} + +// From http://yaml.org/type/float.html, except the regular expression there +// is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. +var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) + +// isOldBool returns whether s is bool notation as defined in YAML 1.1. +// +// We continue to force strings that YAML 1.1 would interpret as booleans to be +// rendered as quotes strings so that the marshalled output valid for YAML 1.1 +// parsing. +func isOldBool(s string) (result bool) { + switch s { + case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", + "n", "N", "no", "No", "NO", "off", "Off", "OFF": + return true + default: + return false + } +} + +func (e *encoder) stringv(tag string, in reflect.Value) { + var style yaml_scalar_style_t + s := in.String() + canUsePlain := true + switch { + case !utf8.ValidString(s): + if tag == binaryTag { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = binaryTag + s = encodeBase64(s) + case tag == "": + // Check to see if it would resolve to a specific + // tag when encoded unquoted. If it doesn't, + // there's no need to quote it. + rtag, _ := resolve("", s) + canUsePlain = rtag == strTag && !(isBase60Float(s) || isOldBool(s)) + } + // Note: it's possible for user code to emit invalid YAML + // if they explicitly specify a tag and a string containing + // text that's incompatible with that tag. + switch { + case strings.Contains(s, "\n"): + if e.flow { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } else { + style = yaml_LITERAL_SCALAR_STYLE + } + case canUsePlain: + style = yaml_PLAIN_SCALAR_STYLE + default: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + e.emitScalar(s, "", tag, style, nil, nil, nil, nil) +} + +func (e *encoder) boolv(tag string, in reflect.Value) { + var s string + if in.Bool() { + s = "true" + } else { + s = "false" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) intv(tag string, in reflect.Value) { + s := strconv.FormatInt(in.Int(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) uintv(tag string, in reflect.Value) { + s := strconv.FormatUint(in.Uint(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) timev(tag string, in reflect.Value) { + t := in.Interface().(time.Time) + s := t.Format(time.RFC3339Nano) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) floatv(tag string, in reflect.Value) { + // Issue #352: When formatting, use the precision of the underlying value + precision := 64 + if in.Kind() == reflect.Float32 { + precision = 32 + } + + s := strconv.FormatFloat(in.Float(), 'g', -1, precision) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) nilv() { + e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t, head, line, foot, tail []byte) { + // TODO Kill this function. Replace all initialize calls by their underlining Go literals. + implicit := tag == "" + if !implicit { + tag = longTag(tag) + } + e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style)) + e.event.head_comment = head + e.event.line_comment = line + e.event.foot_comment = foot + e.event.tail_comment = tail + e.emit() +} + +func (e *encoder) nodev(in reflect.Value) { + e.node(in.Interface().(*Node), "") +} + +func (e *encoder) node(node *Node, tail string) { + // Zero nodes behave as nil. + if node.Kind == 0 && node.IsZero() { + e.nilv() + return + } + + // If the tag was not explicitly requested, and dropping it won't change the + // implicit tag of the value, don't include it in the presentation. + var tag = node.Tag + var stag = shortTag(tag) + var forceQuoting bool + if tag != "" && node.Style&TaggedStyle == 0 { + if node.Kind == ScalarNode { + if stag == strTag && node.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0 { + tag = "" + } else { + rtag, _ := resolve("", node.Value) + if rtag == stag { + tag = "" + } else if stag == strTag { + tag = "" + forceQuoting = true + } + } + } else { + var rtag string + switch node.Kind { + case MappingNode: + rtag = mapTag + case SequenceNode: + rtag = seqTag + } + if rtag == stag { + tag = "" + } + } + } + + switch node.Kind { + case DocumentNode: + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.event.head_comment = []byte(node.HeadComment) + e.emit() + for _, node := range node.Content { + e.node(node, "") + } + yaml_document_end_event_initialize(&e.event, true) + e.event.foot_comment = []byte(node.FootComment) + e.emit() + + case SequenceNode: + style := yaml_BLOCK_SEQUENCE_STYLE + if node.Style&FlowStyle != 0 { + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style)) + e.event.head_comment = []byte(node.HeadComment) + e.emit() + for _, node := range node.Content { + e.node(node, "") + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.event.line_comment = []byte(node.LineComment) + e.event.foot_comment = []byte(node.FootComment) + e.emit() + + case MappingNode: + style := yaml_BLOCK_MAPPING_STYLE + if node.Style&FlowStyle != 0 { + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style) + e.event.tail_comment = []byte(tail) + e.event.head_comment = []byte(node.HeadComment) + e.emit() + + // The tail logic below moves the foot comment of prior keys to the following key, + // since the value for each key may be a nested structure and the foot needs to be + // processed only the entirety of the value is streamed. The last tail is processed + // with the mapping end event. + var tail string + for i := 0; i+1 < len(node.Content); i += 2 { + k := node.Content[i] + foot := k.FootComment + if foot != "" { + kopy := *k + kopy.FootComment = "" + k = &kopy + } + e.node(k, tail) + tail = foot + + v := node.Content[i+1] + e.node(v, "") + } + + yaml_mapping_end_event_initialize(&e.event) + e.event.tail_comment = []byte(tail) + e.event.line_comment = []byte(node.LineComment) + e.event.foot_comment = []byte(node.FootComment) + e.emit() + + case AliasNode: + yaml_alias_event_initialize(&e.event, []byte(node.Value)) + e.event.head_comment = []byte(node.HeadComment) + e.event.line_comment = []byte(node.LineComment) + e.event.foot_comment = []byte(node.FootComment) + e.emit() + + case ScalarNode: + value := node.Value + if !utf8.ValidString(value) { + if stag == binaryTag { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if stag != "" { + failf("cannot marshal invalid UTF-8 data as %s", stag) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = binaryTag + value = encodeBase64(value) + } + + style := yaml_PLAIN_SCALAR_STYLE + switch { + case node.Style&DoubleQuotedStyle != 0: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + case node.Style&SingleQuotedStyle != 0: + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + case node.Style&LiteralStyle != 0: + style = yaml_LITERAL_SCALAR_STYLE + case node.Style&FoldedStyle != 0: + style = yaml_FOLDED_SCALAR_STYLE + case strings.Contains(value, "\n"): + style = yaml_LITERAL_SCALAR_STYLE + case forceQuoting: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + e.emitScalar(value, node.Anchor, tag, style, []byte(node.HeadComment), []byte(node.LineComment), []byte(node.FootComment), []byte(tail)) + default: + failf("cannot encode node with unknown kind %d", node.Kind) + } +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/parserc.go b/scraper-go/vendor/gopkg.in/yaml.v3/parserc.go new file mode 100644 index 00000000..268558a0 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/parserc.go @@ -0,0 +1,1258 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "bytes" +) + +// The parser implements the following grammar: +// +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// implicit_document ::= block_node DOCUMENT-END* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// block_node_or_indentless_sequence ::= +// ALIAS +// | properties (block_content | indentless_block_sequence)? +// | block_content +// | indentless_block_sequence +// block_node ::= ALIAS +// | properties block_content? +// | block_content +// flow_node ::= ALIAS +// | properties flow_content? +// | flow_content +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// block_content ::= block_collection | flow_collection | SCALAR +// flow_content ::= flow_collection | SCALAR +// block_collection ::= block_sequence | block_mapping +// flow_collection ::= flow_sequence | flow_mapping +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// block_mapping ::= BLOCK-MAPPING_START +// ((KEY block_node_or_indentless_sequence?)? +// (VALUE block_node_or_indentless_sequence?)?)* +// BLOCK-END +// flow_sequence ::= FLOW-SEQUENCE-START +// (flow_sequence_entry FLOW-ENTRY)* +// flow_sequence_entry? +// FLOW-SEQUENCE-END +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// flow_mapping ::= FLOW-MAPPING-START +// (flow_mapping_entry FLOW-ENTRY)* +// flow_mapping_entry? +// FLOW-MAPPING-END +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +// Peek the next token in the token queue. +func peek_token(parser *yaml_parser_t) *yaml_token_t { + if parser.token_available || yaml_parser_fetch_more_tokens(parser) { + token := &parser.tokens[parser.tokens_head] + yaml_parser_unfold_comments(parser, token) + return token + } + return nil +} + +// yaml_parser_unfold_comments walks through the comments queue and joins all +// comments behind the position of the provided token into the respective +// top-level comment slices in the parser. +func yaml_parser_unfold_comments(parser *yaml_parser_t, token *yaml_token_t) { + for parser.comments_head < len(parser.comments) && token.start_mark.index >= parser.comments[parser.comments_head].token_mark.index { + comment := &parser.comments[parser.comments_head] + if len(comment.head) > 0 { + if token.typ == yaml_BLOCK_END_TOKEN { + // No heads on ends, so keep comment.head for a follow up token. + break + } + if len(parser.head_comment) > 0 { + parser.head_comment = append(parser.head_comment, '\n') + } + parser.head_comment = append(parser.head_comment, comment.head...) + } + if len(comment.foot) > 0 { + if len(parser.foot_comment) > 0 { + parser.foot_comment = append(parser.foot_comment, '\n') + } + parser.foot_comment = append(parser.foot_comment, comment.foot...) + } + if len(comment.line) > 0 { + if len(parser.line_comment) > 0 { + parser.line_comment = append(parser.line_comment, '\n') + } + parser.line_comment = append(parser.line_comment, comment.line...) + } + *comment = yaml_comment_t{} + parser.comments_head++ + } +} + +// Remove the next token from the queue (must be called after peek_token). +func skip_token(parser *yaml_parser_t) { + parser.token_available = false + parser.tokens_parsed++ + parser.stream_end_produced = parser.tokens[parser.tokens_head].typ == yaml_STREAM_END_TOKEN + parser.tokens_head++ +} + +// Get the next event. +func yaml_parser_parse(parser *yaml_parser_t, event *yaml_event_t) bool { + // Erase the event object. + *event = yaml_event_t{} + + // No events after the end of the stream or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR || parser.state == yaml_PARSE_END_STATE { + return true + } + + // Generate the next event. + return yaml_parser_state_machine(parser, event) +} + +// Set parser error. +func yaml_parser_set_parser_error(parser *yaml_parser_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +func yaml_parser_set_parser_error_context(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +// State dispatcher. +func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool { + //trace("yaml_parser_state_machine", "state:", parser.state.String()) + + switch parser.state { + case yaml_PARSE_STREAM_START_STATE: + return yaml_parser_parse_stream_start(parser, event) + + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, true) + + case yaml_PARSE_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, false) + + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return yaml_parser_parse_document_content(parser, event) + + case yaml_PARSE_DOCUMENT_END_STATE: + return yaml_parser_parse_document_end(parser, event) + + case yaml_PARSE_BLOCK_NODE_STATE: + return yaml_parser_parse_node(parser, event, true, false) + + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return yaml_parser_parse_node(parser, event, true, true) + + case yaml_PARSE_FLOW_NODE_STATE: + return yaml_parser_parse_node(parser, event, false, false) + + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, true) + + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, false) + + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_indentless_sequence_entry(parser, event) + + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, true) + + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, false) + + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return yaml_parser_parse_block_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, true) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, false) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_key(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_end(parser, event) + + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, true) + + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, true) + + default: + panic("invalid parser state") + } +} + +// Parse the production: +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ +func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_STREAM_START_TOKEN { + return yaml_parser_set_parser_error(parser, "did not find expected ", token.start_mark) + } + parser.state = yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + encoding: token.encoding, + } + skip_token(parser) + return true +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* +func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { + + token := peek_token(parser) + if token == nil { + return false + } + + // Parse extra document end indicators. + if !implicit { + for token.typ == yaml_DOCUMENT_END_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + if implicit && token.typ != yaml_VERSION_DIRECTIVE_TOKEN && + token.typ != yaml_TAG_DIRECTIVE_TOKEN && + token.typ != yaml_DOCUMENT_START_TOKEN && + token.typ != yaml_STREAM_END_TOKEN { + // Parse an implicit document. + if !yaml_parser_process_directives(parser, nil, nil) { + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_BLOCK_NODE_STATE + + var head_comment []byte + if len(parser.head_comment) > 0 { + // [Go] Scan the header comment backwards, and if an empty line is found, break + // the header so the part before the last empty line goes into the + // document header, while the bottom of it goes into a follow up event. + for i := len(parser.head_comment) - 1; i > 0; i-- { + if parser.head_comment[i] == '\n' { + if i == len(parser.head_comment)-1 { + head_comment = parser.head_comment[:i] + parser.head_comment = parser.head_comment[i+1:] + break + } else if parser.head_comment[i-1] == '\n' { + head_comment = parser.head_comment[:i-1] + parser.head_comment = parser.head_comment[i+1:] + break + } + } + } + } + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + + head_comment: head_comment, + } + + } else if token.typ != yaml_STREAM_END_TOKEN { + // Parse an explicit document. + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + start_mark := token.start_mark + if !yaml_parser_process_directives(parser, &version_directive, &tag_directives) { + return false + } + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_DOCUMENT_START_TOKEN { + yaml_parser_set_parser_error(parser, + "did not find expected ", token.start_mark) + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_DOCUMENT_CONTENT_STATE + end_mark := token.end_mark + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: false, + } + skip_token(parser) + + } else { + // Parse the stream end. + parser.state = yaml_PARSE_END_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + } + + return true +} + +// Parse the productions: +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** +// +func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN || + token.typ == yaml_TAG_DIRECTIVE_TOKEN || + token.typ == yaml_DOCUMENT_START_TOKEN || + token.typ == yaml_DOCUMENT_END_TOKEN || + token.typ == yaml_STREAM_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + return yaml_parser_process_empty_scalar(parser, event, + token.start_mark) + } + return yaml_parser_parse_node(parser, event, true, false) +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// +func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + start_mark := token.start_mark + end_mark := token.start_mark + + implicit := true + if token.typ == yaml_DOCUMENT_END_TOKEN { + end_mark = token.end_mark + skip_token(parser) + implicit = false + } + + parser.tag_directives = parser.tag_directives[:0] + + parser.state = yaml_PARSE_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + start_mark: start_mark, + end_mark: end_mark, + implicit: implicit, + } + yaml_parser_set_event_comments(parser, event) + if len(event.head_comment) > 0 && len(event.foot_comment) == 0 { + event.foot_comment = event.head_comment + event.head_comment = nil + } + return true +} + +func yaml_parser_set_event_comments(parser *yaml_parser_t, event *yaml_event_t) { + event.head_comment = parser.head_comment + event.line_comment = parser.line_comment + event.foot_comment = parser.foot_comment + parser.head_comment = nil + parser.line_comment = nil + parser.foot_comment = nil + parser.tail_comment = nil + parser.stem_comment = nil +} + +// Parse the productions: +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** +func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { + //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_ALIAS_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + anchor: token.value, + } + yaml_parser_set_event_comments(parser, event) + skip_token(parser) + return true + } + + start_mark := token.start_mark + end_mark := token.start_mark + + var tag_token bool + var tag_handle, tag_suffix, anchor []byte + var tag_mark yaml_mark_t + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + start_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } else if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + start_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + var tag []byte + if tag_token { + if len(tag_handle) == 0 { + tag = tag_suffix + tag_suffix = nil + } else { + for i := range parser.tag_directives { + if bytes.Equal(parser.tag_directives[i].handle, tag_handle) { + tag = append([]byte(nil), parser.tag_directives[i].prefix...) + tag = append(tag, tag_suffix...) + break + } + } + if len(tag) == 0 { + yaml_parser_set_parser_error_context(parser, + "while parsing a node", start_mark, + "found undefined tag handle", tag_mark) + return false + } + } + } + + implicit := len(tag) == 0 + if indentless_sequence && token.typ == yaml_BLOCK_ENTRY_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_SCALAR_TOKEN { + var plain_implicit, quoted_implicit bool + end_mark = token.end_mark + if (len(tag) == 0 && token.style == yaml_PLAIN_SCALAR_STYLE) || (len(tag) == 1 && tag[0] == '!') { + plain_implicit = true + } else if len(tag) == 0 { + quoted_implicit = true + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + value: token.value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(token.style), + } + yaml_parser_set_event_comments(parser, event) + skip_token(parser) + return true + } + if token.typ == yaml_FLOW_SEQUENCE_START_TOKEN { + // [Go] Some of the events below can be merged as they differ only on style. + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_SEQUENCE_STYLE), + } + yaml_parser_set_event_comments(parser, event) + return true + } + if token.typ == yaml_FLOW_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + yaml_parser_set_event_comments(parser, event) + return true + } + if block && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + if parser.stem_comment != nil { + event.head_comment = parser.stem_comment + parser.stem_comment = nil + } + return true + } + if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), + } + if parser.stem_comment != nil { + event.head_comment = parser.stem_comment + parser.stem_comment = nil + } + return true + } + if len(anchor) > 0 || len(tag) > 0 { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + quoted_implicit: false, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true + } + + context := "while parsing a flow node" + if block { + context = "while parsing a block node" + } + yaml_parser_set_parser_error_context(parser, context, start_mark, + "did not find expected node content", token.start_mark) + return false +} + +// Parse the productions: +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* +// +func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + if token == nil { + return false + } + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + prior_head_len := len(parser.head_comment) + skip_token(parser) + yaml_parser_split_stem_comment(parser, prior_head_len) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } else { + parser.state = yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } + if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block collection", context_mark, + "did not find expected '-' indicator", token.start_mark) +} + +// Parse the productions: +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * +func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + prior_head_len := len(parser.head_comment) + skip_token(parser) + yaml_parser_split_stem_comment(parser, prior_head_len) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && + token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be token.end_mark? + } + return true +} + +// Split stem comment from head comment. +// +// When a sequence or map is found under a sequence entry, the former head comment +// is assigned to the underlying sequence or map as a whole, not the individual +// sequence or map entry as would be expected otherwise. To handle this case the +// previous head comment is moved aside as the stem comment. +func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) { + if stem_len == 0 { + return + } + + token := peek_token(parser) + if token == nil || token.typ != yaml_BLOCK_SEQUENCE_START_TOKEN && token.typ != yaml_BLOCK_MAPPING_START_TOKEN { + return + } + + parser.stem_comment = parser.head_comment[:stem_len] + if len(parser.head_comment) == stem_len { + parser.head_comment = nil + } else { + // Copy suffix to prevent very strange bugs if someone ever appends + // further bytes to the prefix in the stem_comment slice above. + parser.head_comment = append([]byte(nil), parser.head_comment[stem_len+1:]...) + } +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* +// +// BLOCK-END +// ********* +// +func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + if token == nil { + return false + } + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + // [Go] A tail comment was left from the prior mapping value processed. Emit an event + // as it needs to be processed with that value and not the following key. + if len(parser.tail_comment) > 0 { + *event = yaml_event_t{ + typ: yaml_TAIL_COMMENT_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + foot_comment: parser.tail_comment, + } + parser.tail_comment = nil + return true + } + + if token.typ == yaml_KEY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } else { + parser.state = yaml_PARSE_BLOCK_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } else if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + yaml_parser_set_event_comments(parser, event) + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block mapping", context_mark, + "did not find expected key", token.start_mark) +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? +// +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END +// +// +func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + if token == nil { + return false + } + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow sequence", context_mark, + "did not find expected ',' or ']'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + implicit: true, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + skip_token(parser) + return true + } else if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + yaml_parser_set_event_comments(parser, event) + + skip_token(parser) + return true +} + +// +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + mark := token.end_mark + skip_token(parser) + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be end_mark? + } + return true +} + +// Parse the productions: +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * +// +func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow mapping", context_mark, + "did not find expected ',' or '}'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } else { + parser.state = yaml_PARSE_FLOW_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + } else if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + yaml_parser_set_event_comments(parser, event) + skip_token(parser) + return true +} + +// Parse the productions: +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * +// +func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { + token := peek_token(parser) + if token == nil { + return false + } + if empty { + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Generate an empty scalar event. +func yaml_parser_process_empty_scalar(parser *yaml_parser_t, event *yaml_event_t, mark yaml_mark_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: mark, + end_mark: mark, + value: nil, // Empty + implicit: true, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true +} + +var default_tag_directives = []yaml_tag_directive_t{ + {[]byte("!"), []byte("!")}, + {[]byte("!!"), []byte("tag:yaml.org,2002:")}, +} + +// Parse directives. +func yaml_parser_process_directives(parser *yaml_parser_t, + version_directive_ref **yaml_version_directive_t, + tag_directives_ref *[]yaml_tag_directive_t) bool { + + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + + token := peek_token(parser) + if token == nil { + return false + } + + for token.typ == yaml_VERSION_DIRECTIVE_TOKEN || token.typ == yaml_TAG_DIRECTIVE_TOKEN { + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN { + if version_directive != nil { + yaml_parser_set_parser_error(parser, + "found duplicate %YAML directive", token.start_mark) + return false + } + if token.major != 1 || token.minor != 1 { + yaml_parser_set_parser_error(parser, + "found incompatible YAML document", token.start_mark) + return false + } + version_directive = &yaml_version_directive_t{ + major: token.major, + minor: token.minor, + } + } else if token.typ == yaml_TAG_DIRECTIVE_TOKEN { + value := yaml_tag_directive_t{ + handle: token.value, + prefix: token.prefix, + } + if !yaml_parser_append_tag_directive(parser, value, false, token.start_mark) { + return false + } + tag_directives = append(tag_directives, value) + } + + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + + for i := range default_tag_directives { + if !yaml_parser_append_tag_directive(parser, default_tag_directives[i], true, token.start_mark) { + return false + } + } + + if version_directive_ref != nil { + *version_directive_ref = version_directive + } + if tag_directives_ref != nil { + *tag_directives_ref = tag_directives + } + return true +} + +// Append a tag directive to the directives stack. +func yaml_parser_append_tag_directive(parser *yaml_parser_t, value yaml_tag_directive_t, allow_duplicates bool, mark yaml_mark_t) bool { + for i := range parser.tag_directives { + if bytes.Equal(value.handle, parser.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_parser_set_parser_error(parser, "found duplicate %TAG directive", mark) + } + } + + // [Go] I suspect the copy is unnecessary. This was likely done + // because there was no way to track ownership of the data. + value_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(value_copy.handle, value.handle) + copy(value_copy.prefix, value.prefix) + parser.tag_directives = append(parser.tag_directives, value_copy) + return true +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/readerc.go b/scraper-go/vendor/gopkg.in/yaml.v3/readerc.go new file mode 100644 index 00000000..b7de0a89 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/readerc.go @@ -0,0 +1,434 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "io" +) + +// Set the reader error and return 0. +func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool { + parser.error = yaml_READER_ERROR + parser.problem = problem + parser.problem_offset = offset + parser.problem_value = value + return false +} + +// Byte order marks. +const ( + bom_UTF8 = "\xef\xbb\xbf" + bom_UTF16LE = "\xff\xfe" + bom_UTF16BE = "\xfe\xff" +) + +// Determine the input stream encoding by checking the BOM symbol. If no BOM is +// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure. +func yaml_parser_determine_encoding(parser *yaml_parser_t) bool { + // Ensure that we had enough bytes in the raw buffer. + for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 { + if !yaml_parser_update_raw_buffer(parser) { + return false + } + } + + // Determine the encoding. + buf := parser.raw_buffer + pos := parser.raw_buffer_pos + avail := len(buf) - pos + if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] { + parser.encoding = yaml_UTF16LE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] { + parser.encoding = yaml_UTF16BE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] { + parser.encoding = yaml_UTF8_ENCODING + parser.raw_buffer_pos += 3 + parser.offset += 3 + } else { + parser.encoding = yaml_UTF8_ENCODING + } + return true +} + +// Update the raw buffer. +func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool { + size_read := 0 + + // Return if the raw buffer is full. + if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) { + return true + } + + // Return on EOF. + if parser.eof { + return true + } + + // Move the remaining bytes in the raw buffer to the beginning. + if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) { + copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:]) + } + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos] + parser.raw_buffer_pos = 0 + + // Call the read handler to fill the buffer. + size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)]) + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read] + if err == io.EOF { + parser.eof = true + } else if err != nil { + return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1) + } + return true +} + +// Ensure that the buffer contains at least `length` characters. +// Return true on success, false on failure. +// +// The length is supposed to be significantly less that the buffer size. +func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool { + if parser.read_handler == nil { + panic("read handler must be set") + } + + // [Go] This function was changed to guarantee the requested length size at EOF. + // The fact we need to do this is pretty awful, but the description above implies + // for that to be the case, and there are tests + + // If the EOF flag is set and the raw buffer is empty, do nothing. + if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) { + // [Go] ACTUALLY! Read the documentation of this function above. + // This is just broken. To return true, we need to have the + // given length in the buffer. Not doing that means every single + // check that calls this function to make sure the buffer has a + // given length is Go) panicking; or C) accessing invalid memory. + //return true + } + + // Return if the buffer contains enough characters. + if parser.unread >= length { + return true + } + + // Determine the input encoding if it is not known yet. + if parser.encoding == yaml_ANY_ENCODING { + if !yaml_parser_determine_encoding(parser) { + return false + } + } + + // Move the unread characters to the beginning of the buffer. + buffer_len := len(parser.buffer) + if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len { + copy(parser.buffer, parser.buffer[parser.buffer_pos:]) + buffer_len -= parser.buffer_pos + parser.buffer_pos = 0 + } else if parser.buffer_pos == buffer_len { + buffer_len = 0 + parser.buffer_pos = 0 + } + + // Open the whole buffer for writing, and cut it before returning. + parser.buffer = parser.buffer[:cap(parser.buffer)] + + // Fill the buffer until it has enough characters. + first := true + for parser.unread < length { + + // Fill the raw buffer if necessary. + if !first || parser.raw_buffer_pos == len(parser.raw_buffer) { + if !yaml_parser_update_raw_buffer(parser) { + parser.buffer = parser.buffer[:buffer_len] + return false + } + } + first = false + + // Decode the raw buffer. + inner: + for parser.raw_buffer_pos != len(parser.raw_buffer) { + var value rune + var width int + + raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos + + // Decode the next character. + switch parser.encoding { + case yaml_UTF8_ENCODING: + // Decode a UTF-8 character. Check RFC 3629 + // (http://www.ietf.org/rfc/rfc3629.txt) for more details. + // + // The following table (taken from the RFC) is used for + // decoding. + // + // Char. number range | UTF-8 octet sequence + // (hexadecimal) | (binary) + // --------------------+------------------------------------ + // 0000 0000-0000 007F | 0xxxxxxx + // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx + // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx + // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Additionally, the characters in the range 0xD800-0xDFFF + // are prohibited as they are reserved for use with UTF-16 + // surrogate pairs. + + // Determine the length of the UTF-8 sequence. + octet := parser.raw_buffer[parser.raw_buffer_pos] + switch { + case octet&0x80 == 0x00: + width = 1 + case octet&0xE0 == 0xC0: + width = 2 + case octet&0xF0 == 0xE0: + width = 3 + case octet&0xF8 == 0xF0: + width = 4 + default: + // The leading octet is invalid. + return yaml_parser_set_reader_error(parser, + "invalid leading UTF-8 octet", + parser.offset, int(octet)) + } + + // Check if the raw buffer contains an incomplete character. + if width > raw_unread { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-8 octet sequence", + parser.offset, -1) + } + break inner + } + + // Decode the leading octet. + switch { + case octet&0x80 == 0x00: + value = rune(octet & 0x7F) + case octet&0xE0 == 0xC0: + value = rune(octet & 0x1F) + case octet&0xF0 == 0xE0: + value = rune(octet & 0x0F) + case octet&0xF8 == 0xF0: + value = rune(octet & 0x07) + default: + value = 0 + } + + // Check and decode the trailing octets. + for k := 1; k < width; k++ { + octet = parser.raw_buffer[parser.raw_buffer_pos+k] + + // Check if the octet is valid. + if (octet & 0xC0) != 0x80 { + return yaml_parser_set_reader_error(parser, + "invalid trailing UTF-8 octet", + parser.offset+k, int(octet)) + } + + // Decode the octet. + value = (value << 6) + rune(octet&0x3F) + } + + // Check the length of the sequence against the value. + switch { + case width == 1: + case width == 2 && value >= 0x80: + case width == 3 && value >= 0x800: + case width == 4 && value >= 0x10000: + default: + return yaml_parser_set_reader_error(parser, + "invalid length of a UTF-8 sequence", + parser.offset, -1) + } + + // Check the range of the value. + if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF { + return yaml_parser_set_reader_error(parser, + "invalid Unicode character", + parser.offset, int(value)) + } + + case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING: + var low, high int + if parser.encoding == yaml_UTF16LE_ENCODING { + low, high = 0, 1 + } else { + low, high = 1, 0 + } + + // The UTF-16 encoding is not as simple as one might + // naively think. Check RFC 2781 + // (http://www.ietf.org/rfc/rfc2781.txt). + // + // Normally, two subsequent bytes describe a Unicode + // character. However a special technique (called a + // surrogate pair) is used for specifying character + // values larger than 0xFFFF. + // + // A surrogate pair consists of two pseudo-characters: + // high surrogate area (0xD800-0xDBFF) + // low surrogate area (0xDC00-0xDFFF) + // + // The following formulas are used for decoding + // and encoding characters using surrogate pairs: + // + // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF) + // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF) + // W1 = 110110yyyyyyyyyy + // W2 = 110111xxxxxxxxxx + // + // where U is the character value, W1 is the high surrogate + // area, W2 is the low surrogate area. + + // Check for incomplete UTF-16 character. + if raw_unread < 2 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 character", + parser.offset, -1) + } + break inner + } + + // Get the character. + value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8) + + // Check for unexpected low surrogate area. + if value&0xFC00 == 0xDC00 { + return yaml_parser_set_reader_error(parser, + "unexpected low surrogate area", + parser.offset, int(value)) + } + + // Check for a high surrogate area. + if value&0xFC00 == 0xD800 { + width = 4 + + // Check for incomplete surrogate pair. + if raw_unread < 4 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 surrogate pair", + parser.offset, -1) + } + break inner + } + + // Get the next character. + value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8) + + // Check for a low surrogate area. + if value2&0xFC00 != 0xDC00 { + return yaml_parser_set_reader_error(parser, + "expected low surrogate area", + parser.offset+2, int(value2)) + } + + // Generate the value of the surrogate pair. + value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF) + } else { + width = 2 + } + + default: + panic("impossible") + } + + // Check if the character is in the allowed range: + // #x9 | #xA | #xD | [#x20-#x7E] (8 bit) + // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) + // | [#x10000-#x10FFFF] (32 bit) + switch { + case value == 0x09: + case value == 0x0A: + case value == 0x0D: + case value >= 0x20 && value <= 0x7E: + case value == 0x85: + case value >= 0xA0 && value <= 0xD7FF: + case value >= 0xE000 && value <= 0xFFFD: + case value >= 0x10000 && value <= 0x10FFFF: + default: + return yaml_parser_set_reader_error(parser, + "control characters are not allowed", + parser.offset, int(value)) + } + + // Move the raw pointers. + parser.raw_buffer_pos += width + parser.offset += width + + // Finally put the character into the buffer. + if value <= 0x7F { + // 0000 0000-0000 007F . 0xxxxxxx + parser.buffer[buffer_len+0] = byte(value) + buffer_len += 1 + } else if value <= 0x7FF { + // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6)) + parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F)) + buffer_len += 2 + } else if value <= 0xFFFF { + // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F)) + buffer_len += 3 + } else { + // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F)) + buffer_len += 4 + } + + parser.unread++ + } + + // On EOF, put NUL into the buffer and return. + if parser.eof { + parser.buffer[buffer_len] = 0 + buffer_len++ + parser.unread++ + break + } + } + // [Go] Read the documentation of this function above. To return true, + // we need to have the given length in the buffer. Not doing that means + // every single check that calls this function to make sure the buffer + // has a given length is Go) panicking; or C) accessing invalid memory. + // This happens here due to the EOF above breaking early. + for buffer_len < length { + parser.buffer[buffer_len] = 0 + buffer_len++ + } + parser.buffer = parser.buffer[:buffer_len] + return true +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/resolve.go b/scraper-go/vendor/gopkg.in/yaml.v3/resolve.go new file mode 100644 index 00000000..64ae8880 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/resolve.go @@ -0,0 +1,326 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "encoding/base64" + "math" + "regexp" + "strconv" + "strings" + "time" +) + +type resolveMapItem struct { + value interface{} + tag string +} + +var resolveTable = make([]byte, 256) +var resolveMap = make(map[string]resolveMapItem) + +func init() { + t := resolveTable + t[int('+')] = 'S' // Sign + t[int('-')] = 'S' + for _, c := range "0123456789" { + t[int(c)] = 'D' // Digit + } + for _, c := range "yYnNtTfFoO~" { + t[int(c)] = 'M' // In map + } + t[int('.')] = '.' // Float (potentially in map) + + var resolveMapList = []struct { + v interface{} + tag string + l []string + }{ + {true, boolTag, []string{"true", "True", "TRUE"}}, + {false, boolTag, []string{"false", "False", "FALSE"}}, + {nil, nullTag, []string{"", "~", "null", "Null", "NULL"}}, + {math.NaN(), floatTag, []string{".nan", ".NaN", ".NAN"}}, + {math.Inf(+1), floatTag, []string{".inf", ".Inf", ".INF"}}, + {math.Inf(+1), floatTag, []string{"+.inf", "+.Inf", "+.INF"}}, + {math.Inf(-1), floatTag, []string{"-.inf", "-.Inf", "-.INF"}}, + {"<<", mergeTag, []string{"<<"}}, + } + + m := resolveMap + for _, item := range resolveMapList { + for _, s := range item.l { + m[s] = resolveMapItem{item.v, item.tag} + } + } +} + +const ( + nullTag = "!!null" + boolTag = "!!bool" + strTag = "!!str" + intTag = "!!int" + floatTag = "!!float" + timestampTag = "!!timestamp" + seqTag = "!!seq" + mapTag = "!!map" + binaryTag = "!!binary" + mergeTag = "!!merge" +) + +var longTags = make(map[string]string) +var shortTags = make(map[string]string) + +func init() { + for _, stag := range []string{nullTag, boolTag, strTag, intTag, floatTag, timestampTag, seqTag, mapTag, binaryTag, mergeTag} { + ltag := longTag(stag) + longTags[stag] = ltag + shortTags[ltag] = stag + } +} + +const longTagPrefix = "tag:yaml.org,2002:" + +func shortTag(tag string) string { + if strings.HasPrefix(tag, longTagPrefix) { + if stag, ok := shortTags[tag]; ok { + return stag + } + return "!!" + tag[len(longTagPrefix):] + } + return tag +} + +func longTag(tag string) string { + if strings.HasPrefix(tag, "!!") { + if ltag, ok := longTags[tag]; ok { + return ltag + } + return longTagPrefix + tag[2:] + } + return tag +} + +func resolvableTag(tag string) bool { + switch tag { + case "", strTag, boolTag, intTag, floatTag, nullTag, timestampTag: + return true + } + return false +} + +var yamlStyleFloat = regexp.MustCompile(`^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$`) + +func resolve(tag string, in string) (rtag string, out interface{}) { + tag = shortTag(tag) + if !resolvableTag(tag) { + return tag, in + } + + defer func() { + switch tag { + case "", rtag, strTag, binaryTag: + return + case floatTag: + if rtag == intTag { + switch v := out.(type) { + case int64: + rtag = floatTag + out = float64(v) + return + case int: + rtag = floatTag + out = float64(v) + return + } + } + } + failf("cannot decode %s `%s` as a %s", shortTag(rtag), in, shortTag(tag)) + }() + + // Any data is accepted as a !!str or !!binary. + // Otherwise, the prefix is enough of a hint about what it might be. + hint := byte('N') + if in != "" { + hint = resolveTable[in[0]] + } + if hint != 0 && tag != strTag && tag != binaryTag { + // Handle things we can lookup in a map. + if item, ok := resolveMap[in]; ok { + return item.tag, item.value + } + + // Base 60 floats are a bad idea, were dropped in YAML 1.2, and + // are purposefully unsupported here. They're still quoted on + // the way out for compatibility with other parser, though. + + switch hint { + case 'M': + // We've already checked the map above. + + case '.': + // Not in the map, so maybe a normal float. + floatv, err := strconv.ParseFloat(in, 64) + if err == nil { + return floatTag, floatv + } + + case 'D', 'S': + // Int, float, or timestamp. + // Only try values as a timestamp if the value is unquoted or there's an explicit + // !!timestamp tag. + if tag == "" || tag == timestampTag { + t, ok := parseTimestamp(in) + if ok { + return timestampTag, t + } + } + + plain := strings.Replace(in, "_", "", -1) + intv, err := strconv.ParseInt(plain, 0, 64) + if err == nil { + if intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + uintv, err := strconv.ParseUint(plain, 0, 64) + if err == nil { + return intTag, uintv + } + if yamlStyleFloat.MatchString(plain) { + floatv, err := strconv.ParseFloat(plain, 64) + if err == nil { + return floatTag, floatv + } + } + if strings.HasPrefix(plain, "0b") { + intv, err := strconv.ParseInt(plain[2:], 2, 64) + if err == nil { + if intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 2, 64) + if err == nil { + return intTag, uintv + } + } else if strings.HasPrefix(plain, "-0b") { + intv, err := strconv.ParseInt("-"+plain[3:], 2, 64) + if err == nil { + if true || intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + } + // Octals as introduced in version 1.2 of the spec. + // Octals from the 1.1 spec, spelled as 0777, are still + // decoded by default in v3 as well for compatibility. + // May be dropped in v4 depending on how usage evolves. + if strings.HasPrefix(plain, "0o") { + intv, err := strconv.ParseInt(plain[2:], 8, 64) + if err == nil { + if intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 8, 64) + if err == nil { + return intTag, uintv + } + } else if strings.HasPrefix(plain, "-0o") { + intv, err := strconv.ParseInt("-"+plain[3:], 8, 64) + if err == nil { + if true || intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + } + default: + panic("internal error: missing handler for resolver table: " + string(rune(hint)) + " (with " + in + ")") + } + } + return strTag, in +} + +// encodeBase64 encodes s as base64 that is broken up into multiple lines +// as appropriate for the resulting length. +func encodeBase64(s string) string { + const lineLen = 70 + encLen := base64.StdEncoding.EncodedLen(len(s)) + lines := encLen/lineLen + 1 + buf := make([]byte, encLen*2+lines) + in := buf[0:encLen] + out := buf[encLen:] + base64.StdEncoding.Encode(in, []byte(s)) + k := 0 + for i := 0; i < len(in); i += lineLen { + j := i + lineLen + if j > len(in) { + j = len(in) + } + k += copy(out[k:], in[i:j]) + if lines > 1 { + out[k] = '\n' + k++ + } + } + return string(out[:k]) +} + +// This is a subset of the formats allowed by the regular expression +// defined at http://yaml.org/type/timestamp.html. +var allowedTimestampFormats = []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + // Notable exception: time.Parse cannot handle: "2001-12-14 21:59:43.10 -5" + // from the set of examples. +} + +// parseTimestamp parses s as a timestamp string and +// returns the timestamp and reports whether it succeeded. +// Timestamp formats are defined at http://yaml.org/type/timestamp.html +func parseTimestamp(s string) (time.Time, bool) { + // TODO write code to check all the formats supported by + // http://yaml.org/type/timestamp.html instead of using time.Parse. + + // Quick check: all date formats start with YYYY-. + i := 0 + for ; i < len(s); i++ { + if c := s[i]; c < '0' || c > '9' { + break + } + } + if i != 4 || i == len(s) || s[i] != '-' { + return time.Time{}, false + } + for _, format := range allowedTimestampFormats { + if t, err := time.Parse(format, s); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/scannerc.go b/scraper-go/vendor/gopkg.in/yaml.v3/scannerc.go new file mode 100644 index 00000000..ca007010 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/scannerc.go @@ -0,0 +1,3038 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "bytes" + "fmt" +) + +// Introduction +// ************ +// +// The following notes assume that you are familiar with the YAML specification +// (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in +// some cases we are less restrictive that it requires. +// +// The process of transforming a YAML stream into a sequence of events is +// divided on two steps: Scanning and Parsing. +// +// The Scanner transforms the input stream into a sequence of tokens, while the +// parser transform the sequence of tokens produced by the Scanner into a +// sequence of parsing events. +// +// The Scanner is rather clever and complicated. The Parser, on the contrary, +// is a straightforward implementation of a recursive-descendant parser (or, +// LL(1) parser, as it is usually called). +// +// Actually there are two issues of Scanning that might be called "clever", the +// rest is quite straightforward. The issues are "block collection start" and +// "simple keys". Both issues are explained below in details. +// +// Here the Scanning step is explained and implemented. We start with the list +// of all the tokens produced by the Scanner together with short descriptions. +// +// Now, tokens: +// +// STREAM-START(encoding) # The stream start. +// STREAM-END # The stream end. +// VERSION-DIRECTIVE(major,minor) # The '%YAML' directive. +// TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive. +// DOCUMENT-START # '---' +// DOCUMENT-END # '...' +// BLOCK-SEQUENCE-START # Indentation increase denoting a block +// BLOCK-MAPPING-START # sequence or a block mapping. +// BLOCK-END # Indentation decrease. +// FLOW-SEQUENCE-START # '[' +// FLOW-SEQUENCE-END # ']' +// BLOCK-SEQUENCE-START # '{' +// BLOCK-SEQUENCE-END # '}' +// BLOCK-ENTRY # '-' +// FLOW-ENTRY # ',' +// KEY # '?' or nothing (simple keys). +// VALUE # ':' +// ALIAS(anchor) # '*anchor' +// ANCHOR(anchor) # '&anchor' +// TAG(handle,suffix) # '!handle!suffix' +// SCALAR(value,style) # A scalar. +// +// The following two tokens are "virtual" tokens denoting the beginning and the +// end of the stream: +// +// STREAM-START(encoding) +// STREAM-END +// +// We pass the information about the input stream encoding with the +// STREAM-START token. +// +// The next two tokens are responsible for tags: +// +// VERSION-DIRECTIVE(major,minor) +// TAG-DIRECTIVE(handle,prefix) +// +// Example: +// +// %YAML 1.1 +// %TAG ! !foo +// %TAG !yaml! tag:yaml.org,2002: +// --- +// +// The correspoding sequence of tokens: +// +// STREAM-START(utf-8) +// VERSION-DIRECTIVE(1,1) +// TAG-DIRECTIVE("!","!foo") +// TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:") +// DOCUMENT-START +// STREAM-END +// +// Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole +// line. +// +// The document start and end indicators are represented by: +// +// DOCUMENT-START +// DOCUMENT-END +// +// Note that if a YAML stream contains an implicit document (without '---' +// and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be +// produced. +// +// In the following examples, we present whole documents together with the +// produced tokens. +// +// 1. An implicit document: +// +// 'a scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// STREAM-END +// +// 2. An explicit document: +// +// --- +// 'a scalar' +// ... +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// SCALAR("a scalar",single-quoted) +// DOCUMENT-END +// STREAM-END +// +// 3. Several documents in a stream: +// +// 'a scalar' +// --- +// 'another scalar' +// --- +// 'yet another scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// DOCUMENT-START +// SCALAR("another scalar",single-quoted) +// DOCUMENT-START +// SCALAR("yet another scalar",single-quoted) +// STREAM-END +// +// We have already introduced the SCALAR token above. The following tokens are +// used to describe aliases, anchors, tag, and scalars: +// +// ALIAS(anchor) +// ANCHOR(anchor) +// TAG(handle,suffix) +// SCALAR(value,style) +// +// The following series of examples illustrate the usage of these tokens: +// +// 1. A recursive sequence: +// +// &A [ *A ] +// +// Tokens: +// +// STREAM-START(utf-8) +// ANCHOR("A") +// FLOW-SEQUENCE-START +// ALIAS("A") +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A tagged scalar: +// +// !!float "3.14" # A good approximation. +// +// Tokens: +// +// STREAM-START(utf-8) +// TAG("!!","float") +// SCALAR("3.14",double-quoted) +// STREAM-END +// +// 3. Various scalar styles: +// +// --- # Implicit empty plain scalars do not produce tokens. +// --- a plain scalar +// --- 'a single-quoted scalar' +// --- "a double-quoted scalar" +// --- |- +// a literal scalar +// --- >- +// a folded +// scalar +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// DOCUMENT-START +// SCALAR("a plain scalar",plain) +// DOCUMENT-START +// SCALAR("a single-quoted scalar",single-quoted) +// DOCUMENT-START +// SCALAR("a double-quoted scalar",double-quoted) +// DOCUMENT-START +// SCALAR("a literal scalar",literal) +// DOCUMENT-START +// SCALAR("a folded scalar",folded) +// STREAM-END +// +// Now it's time to review collection-related tokens. We will start with +// flow collections: +// +// FLOW-SEQUENCE-START +// FLOW-SEQUENCE-END +// FLOW-MAPPING-START +// FLOW-MAPPING-END +// FLOW-ENTRY +// KEY +// VALUE +// +// The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and +// FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}' +// correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the +// indicators '?' and ':', which are used for denoting mapping keys and values, +// are represented by the KEY and VALUE tokens. +// +// The following examples show flow collections: +// +// 1. A flow sequence: +// +// [item 1, item 2, item 3] +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-SEQUENCE-START +// SCALAR("item 1",plain) +// FLOW-ENTRY +// SCALAR("item 2",plain) +// FLOW-ENTRY +// SCALAR("item 3",plain) +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A flow mapping: +// +// { +// a simple key: a value, # Note that the KEY token is produced. +// ? a complex key: another value, +// } +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// FLOW-ENTRY +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// FLOW-ENTRY +// FLOW-MAPPING-END +// STREAM-END +// +// A simple key is a key which is not denoted by the '?' indicator. Note that +// the Scanner still produce the KEY token whenever it encounters a simple key. +// +// For scanning block collections, the following tokens are used (note that we +// repeat KEY and VALUE here): +// +// BLOCK-SEQUENCE-START +// BLOCK-MAPPING-START +// BLOCK-END +// BLOCK-ENTRY +// KEY +// VALUE +// +// The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation +// increase that precedes a block collection (cf. the INDENT token in Python). +// The token BLOCK-END denote indentation decrease that ends a block collection +// (cf. the DEDENT token in Python). However YAML has some syntax pecularities +// that makes detections of these tokens more complex. +// +// The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators +// '-', '?', and ':' correspondingly. +// +// The following examples show how the tokens BLOCK-SEQUENCE-START, +// BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner: +// +// 1. Block sequences: +// +// - item 1 +// - item 2 +// - +// - item 3.1 +// - item 3.2 +// - +// key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 3.1",plain) +// BLOCK-ENTRY +// SCALAR("item 3.2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Block mappings: +// +// a simple key: a value # The KEY token is produced here. +// ? a complex key +// : another value +// a mapping: +// key 1: value 1 +// key 2: value 2 +// a sequence: +// - item 1 +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// KEY +// SCALAR("a mapping",plain) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML does not always require to start a new block collection from a new +// line. If the current line contains only '-', '?', and ':' indicators, a new +// block collection may start at the current line. The following examples +// illustrate this case: +// +// 1. Collections in a sequence: +// +// - - item 1 +// - item 2 +// - key 1: value 1 +// key 2: value 2 +// - ? complex key +// : complex value +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("complex key") +// VALUE +// SCALAR("complex value") +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Collections in a mapping: +// +// ? a sequence +// : - item 1 +// - item 2 +// ? a mapping +// : key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// KEY +// SCALAR("a mapping",plain) +// VALUE +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML also permits non-indented sequences if they are included into a block +// mapping. In this case, the token BLOCK-SEQUENCE-START is not produced: +// +// key: +// - item 1 # BLOCK-SEQUENCE-START is NOT produced here. +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key",plain) +// VALUE +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// + +// Ensure that the buffer contains the required number of characters. +// Return true on success, false on failure (reader error or memory error). +func cache(parser *yaml_parser_t, length int) bool { + // [Go] This was inlined: !cache(A, B) -> unread < B && !update(A, B) + return parser.unread >= length || yaml_parser_update_buffer(parser, length) +} + +// Advance the buffer pointer. +func skip(parser *yaml_parser_t) { + if !is_blank(parser.buffer, parser.buffer_pos) { + parser.newlines = 0 + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) +} + +func skip_line(parser *yaml_parser_t) { + if is_crlf(parser.buffer, parser.buffer_pos) { + parser.mark.index += 2 + parser.mark.column = 0 + parser.mark.line++ + parser.unread -= 2 + parser.buffer_pos += 2 + parser.newlines++ + } else if is_break(parser.buffer, parser.buffer_pos) { + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) + parser.newlines++ + } +} + +// Copy a character to a string buffer and advance pointers. +func read(parser *yaml_parser_t, s []byte) []byte { + if !is_blank(parser.buffer, parser.buffer_pos) { + parser.newlines = 0 + } + w := width(parser.buffer[parser.buffer_pos]) + if w == 0 { + panic("invalid character sequence") + } + if len(s) == 0 { + s = make([]byte, 0, 32) + } + if w == 1 && len(s)+w <= cap(s) { + s = s[:len(s)+1] + s[len(s)-1] = parser.buffer[parser.buffer_pos] + parser.buffer_pos++ + } else { + s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...) + parser.buffer_pos += w + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + return s +} + +// Copy a line break character to a string buffer and advance pointers. +func read_line(parser *yaml_parser_t, s []byte) []byte { + buf := parser.buffer + pos := parser.buffer_pos + switch { + case buf[pos] == '\r' && buf[pos+1] == '\n': + // CR LF . LF + s = append(s, '\n') + parser.buffer_pos += 2 + parser.mark.index++ + parser.unread-- + case buf[pos] == '\r' || buf[pos] == '\n': + // CR|LF . LF + s = append(s, '\n') + parser.buffer_pos += 1 + case buf[pos] == '\xC2' && buf[pos+1] == '\x85': + // NEL . LF + s = append(s, '\n') + parser.buffer_pos += 2 + case buf[pos] == '\xE2' && buf[pos+1] == '\x80' && (buf[pos+2] == '\xA8' || buf[pos+2] == '\xA9'): + // LS|PS . LS|PS + s = append(s, buf[parser.buffer_pos:pos+3]...) + parser.buffer_pos += 3 + default: + return s + } + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.newlines++ + return s +} + +// Get the next token. +func yaml_parser_scan(parser *yaml_parser_t, token *yaml_token_t) bool { + // Erase the token object. + *token = yaml_token_t{} // [Go] Is this necessary? + + // No tokens after STREAM-END or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR { + return true + } + + // Ensure that the tokens queue contains enough tokens. + if !parser.token_available { + if !yaml_parser_fetch_more_tokens(parser) { + return false + } + } + + // Fetch the next token from the queue. + *token = parser.tokens[parser.tokens_head] + parser.tokens_head++ + parser.tokens_parsed++ + parser.token_available = false + + if token.typ == yaml_STREAM_END_TOKEN { + parser.stream_end_produced = true + } + return true +} + +// Set the scanner error and return false. +func yaml_parser_set_scanner_error(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string) bool { + parser.error = yaml_SCANNER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = parser.mark + return false +} + +func yaml_parser_set_scanner_tag_error(parser *yaml_parser_t, directive bool, context_mark yaml_mark_t, problem string) bool { + context := "while parsing a tag" + if directive { + context = "while parsing a %TAG directive" + } + return yaml_parser_set_scanner_error(parser, context, context_mark, problem) +} + +func trace(args ...interface{}) func() { + pargs := append([]interface{}{"+++"}, args...) + fmt.Println(pargs...) + pargs = append([]interface{}{"---"}, args...) + return func() { fmt.Println(pargs...) } +} + +// Ensure that the tokens queue contains at least one token which can be +// returned to the Parser. +func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { + // While we need more tokens to fetch, do it. + for { + // [Go] The comment parsing logic requires a lookahead of two tokens + // so that foot comments may be parsed in time of associating them + // with the tokens that are parsed before them, and also for line + // comments to be transformed into head comments in some edge cases. + if parser.tokens_head < len(parser.tokens)-2 { + // If a potential simple key is at the head position, we need to fetch + // the next token to disambiguate it. + head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed] + if !ok { + break + } else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok { + return false + } else if !valid { + break + } + } + // Fetch the next token. + if !yaml_parser_fetch_next_token(parser) { + return false + } + } + + parser.token_available = true + return true +} + +// The dispatcher for token fetchers. +func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { + // Ensure that the buffer is initialized. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we just started scanning. Fetch STREAM-START then. + if !parser.stream_start_produced { + return yaml_parser_fetch_stream_start(parser) + } + + scan_mark := parser.mark + + // Eat whitespaces and comments until we reach the next token. + if !yaml_parser_scan_to_next_token(parser) { + return false + } + + // [Go] While unrolling indents, transform the head comments of prior + // indentation levels observed after scan_start into foot comments at + // the respective indexes. + + // Check the indentation level against the current column. + if !yaml_parser_unroll_indent(parser, parser.mark.column, scan_mark) { + return false + } + + // Ensure that the buffer contains at least 4 characters. 4 is the length + // of the longest indicators ('--- ' and '... '). + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + // Is it the end of the stream? + if is_z(parser.buffer, parser.buffer_pos) { + return yaml_parser_fetch_stream_end(parser) + } + + // Is it a directive? + if parser.mark.column == 0 && parser.buffer[parser.buffer_pos] == '%' { + return yaml_parser_fetch_directive(parser) + } + + buf := parser.buffer + pos := parser.buffer_pos + + // Is it the document start indicator? + if parser.mark.column == 0 && buf[pos] == '-' && buf[pos+1] == '-' && buf[pos+2] == '-' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_START_TOKEN) + } + + // Is it the document end indicator? + if parser.mark.column == 0 && buf[pos] == '.' && buf[pos+1] == '.' && buf[pos+2] == '.' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN) + } + + comment_mark := parser.mark + if len(parser.tokens) > 0 && (parser.flow_level == 0 && buf[pos] == ':' || parser.flow_level > 0 && buf[pos] == ',') { + // Associate any following comments with the prior token. + comment_mark = parser.tokens[len(parser.tokens)-1].start_mark + } + defer func() { + if !ok { + return + } + if len(parser.tokens) > 0 && parser.tokens[len(parser.tokens)-1].typ == yaml_BLOCK_ENTRY_TOKEN { + // Sequence indicators alone have no line comments. It becomes + // a head comment for whatever follows. + return + } + if !yaml_parser_scan_line_comment(parser, comment_mark) { + ok = false + return + } + }() + + // Is it the flow sequence start indicator? + if buf[pos] == '[' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN) + } + + // Is it the flow mapping start indicator? + if parser.buffer[parser.buffer_pos] == '{' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_MAPPING_START_TOKEN) + } + + // Is it the flow sequence end indicator? + if parser.buffer[parser.buffer_pos] == ']' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_SEQUENCE_END_TOKEN) + } + + // Is it the flow mapping end indicator? + if parser.buffer[parser.buffer_pos] == '}' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_MAPPING_END_TOKEN) + } + + // Is it the flow entry indicator? + if parser.buffer[parser.buffer_pos] == ',' { + return yaml_parser_fetch_flow_entry(parser) + } + + // Is it the block entry indicator? + if parser.buffer[parser.buffer_pos] == '-' && is_blankz(parser.buffer, parser.buffer_pos+1) { + return yaml_parser_fetch_block_entry(parser) + } + + // Is it the key indicator? + if parser.buffer[parser.buffer_pos] == '?' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_key(parser) + } + + // Is it the value indicator? + if parser.buffer[parser.buffer_pos] == ':' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_value(parser) + } + + // Is it an alias? + if parser.buffer[parser.buffer_pos] == '*' { + return yaml_parser_fetch_anchor(parser, yaml_ALIAS_TOKEN) + } + + // Is it an anchor? + if parser.buffer[parser.buffer_pos] == '&' { + return yaml_parser_fetch_anchor(parser, yaml_ANCHOR_TOKEN) + } + + // Is it a tag? + if parser.buffer[parser.buffer_pos] == '!' { + return yaml_parser_fetch_tag(parser) + } + + // Is it a literal scalar? + if parser.buffer[parser.buffer_pos] == '|' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, true) + } + + // Is it a folded scalar? + if parser.buffer[parser.buffer_pos] == '>' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, false) + } + + // Is it a single-quoted scalar? + if parser.buffer[parser.buffer_pos] == '\'' { + return yaml_parser_fetch_flow_scalar(parser, true) + } + + // Is it a double-quoted scalar? + if parser.buffer[parser.buffer_pos] == '"' { + return yaml_parser_fetch_flow_scalar(parser, false) + } + + // Is it a plain scalar? + // + // A plain scalar may start with any non-blank characters except + // + // '-', '?', ':', ',', '[', ']', '{', '}', + // '#', '&', '*', '!', '|', '>', '\'', '\"', + // '%', '@', '`'. + // + // In the block context (and, for the '-' indicator, in the flow context + // too), it may also start with the characters + // + // '-', '?', ':' + // + // if it is followed by a non-space character. + // + // The last rule is more restrictive than the specification requires. + // [Go] TODO Make this logic more reasonable. + //switch parser.buffer[parser.buffer_pos] { + //case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`': + //} + if !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '-' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}' || parser.buffer[parser.buffer_pos] == '#' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '*' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '|' || + parser.buffer[parser.buffer_pos] == '>' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '"' || parser.buffer[parser.buffer_pos] == '%' || + parser.buffer[parser.buffer_pos] == '@' || parser.buffer[parser.buffer_pos] == '`') || + (parser.buffer[parser.buffer_pos] == '-' && !is_blank(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level == 0 && + (parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':') && + !is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_plain_scalar(parser) + } + + // If we don't determine the token type so far, it is an error. + return yaml_parser_set_scanner_error(parser, + "while scanning for the next token", parser.mark, + "found character that cannot start any token") +} + +func yaml_simple_key_is_valid(parser *yaml_parser_t, simple_key *yaml_simple_key_t) (valid, ok bool) { + if !simple_key.possible { + return false, true + } + + // The 1.2 specification says: + // + // "If the ? indicator is omitted, parsing needs to see past the + // implicit key to recognize it as such. To limit the amount of + // lookahead required, the “:” indicator must appear at most 1024 + // Unicode characters beyond the start of the key. In addition, the key + // is restricted to a single line." + // + if simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index { + // Check if the potential simple key to be removed is required. + if simple_key.required { + return false, yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false + return false, true + } + return true, true +} + +// Check if a simple key may start at the current position and add it if +// needed. +func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { + // A simple key is required at the current position if the scanner is in + // the block context and the current column coincides with the indentation + // level. + + required := parser.flow_level == 0 && parser.indent == parser.mark.column + + // + // If the current position may start a simple key, save it. + // + if parser.simple_key_allowed { + simple_key := yaml_simple_key_t{ + possible: true, + required: required, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + } + + if !yaml_parser_remove_simple_key(parser) { + return false + } + parser.simple_keys[len(parser.simple_keys)-1] = simple_key + parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1 + } + return true +} + +// Remove a potential simple key at the current flow level. +func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { + i := len(parser.simple_keys) - 1 + if parser.simple_keys[i].possible { + // If the key is required, it is an error. + if parser.simple_keys[i].required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", parser.simple_keys[i].mark, + "could not find expected ':'") + } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number) + } + return true +} + +// max_flow_level limits the flow_level +const max_flow_level = 10000 + +// Increase the flow level and resize the simple key list if needed. +func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { + // Reset the simple key on the next level. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{ + possible: false, + required: false, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + }) + + // Increase the flow level. + parser.flow_level++ + if parser.flow_level > max_flow_level { + return yaml_parser_set_scanner_error(parser, + "while increasing flow level", parser.simple_keys[len(parser.simple_keys)-1].mark, + fmt.Sprintf("exceeded max depth of %d", max_flow_level)) + } + return true +} + +// Decrease the flow level. +func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { + if parser.flow_level > 0 { + parser.flow_level-- + last := len(parser.simple_keys) - 1 + delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number) + parser.simple_keys = parser.simple_keys[:last] + } + return true +} + +// max_indents limits the indents stack size +const max_indents = 10000 + +// Push the current indentation level to the stack and set the new level +// the current column is greater than the indentation level. In this case, +// append or insert the specified token into the token queue. +func yaml_parser_roll_indent(parser *yaml_parser_t, column, number int, typ yaml_token_type_t, mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + if parser.indent < column { + // Push the current indentation level to the stack and set the new + // indentation level. + parser.indents = append(parser.indents, parser.indent) + parser.indent = column + if len(parser.indents) > max_indents { + return yaml_parser_set_scanner_error(parser, + "while increasing indent level", parser.simple_keys[len(parser.simple_keys)-1].mark, + fmt.Sprintf("exceeded max depth of %d", max_indents)) + } + + // Create a token and insert it into the queue. + token := yaml_token_t{ + typ: typ, + start_mark: mark, + end_mark: mark, + } + if number > -1 { + number -= parser.tokens_parsed + } + yaml_insert_token(parser, number, &token) + } + return true +} + +// Pop indentation levels from the indents stack until the current level +// becomes less or equal to the column. For each indentation level, append +// the BLOCK-END token. +func yaml_parser_unroll_indent(parser *yaml_parser_t, column int, scan_mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + block_mark := scan_mark + block_mark.index-- + + // Loop through the indentation levels in the stack. + for parser.indent > column { + + // [Go] Reposition the end token before potential following + // foot comments of parent blocks. For that, search + // backwards for recent comments that were at the same + // indent as the block that is ending now. + stop_index := block_mark.index + for i := len(parser.comments) - 1; i >= 0; i-- { + comment := &parser.comments[i] + + if comment.end_mark.index < stop_index { + // Don't go back beyond the start of the comment/whitespace scan, unless column < 0. + // If requested indent column is < 0, then the document is over and everything else + // is a foot anyway. + break + } + if comment.start_mark.column == parser.indent+1 { + // This is a good match. But maybe there's a former comment + // at that same indent level, so keep searching. + block_mark = comment.start_mark + } + + // While the end of the former comment matches with + // the start of the following one, we know there's + // nothing in between and scanning is still safe. + stop_index = comment.scan_mark.index + } + + // Create a token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_END_TOKEN, + start_mark: block_mark, + end_mark: block_mark, + } + yaml_insert_token(parser, -1, &token) + + // Pop the indentation level. + parser.indent = parser.indents[len(parser.indents)-1] + parser.indents = parser.indents[:len(parser.indents)-1] + } + return true +} + +// Initialize the scanner and produce the STREAM-START token. +func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { + + // Set the initial indentation. + parser.indent = -1 + + // Initialize the simple key stack. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + parser.simple_keys_by_tok = make(map[int]int) + + // A simple key is allowed at the beginning of the stream. + parser.simple_key_allowed = true + + // We have started. + parser.stream_start_produced = true + + // Create the STREAM-START token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_START_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + encoding: parser.encoding, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the STREAM-END token and shut down the scanner. +func yaml_parser_fetch_stream_end(parser *yaml_parser_t) bool { + + // Force new line. + if parser.mark.column != 0 { + parser.mark.column = 0 + parser.mark.line++ + } + + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1, parser.mark) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the STREAM-END token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token. +func yaml_parser_fetch_directive(parser *yaml_parser_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1, parser.mark) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. + token := yaml_token_t{} + if !yaml_parser_scan_directive(parser, &token) { + return false + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the DOCUMENT-START or DOCUMENT-END token. +func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1, parser.mark) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Consume the token. + start_mark := parser.mark + + skip(parser) + skip(parser) + skip(parser) + + end_mark := parser.mark + + // Create the DOCUMENT-START or DOCUMENT-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token. +func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool { + + // The indicators '[' and '{' may start a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // Increase the flow level. + if !yaml_parser_increase_flow_level(parser) { + return false + } + + // A simple key may follow the indicators '[' and '{'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token. +func yaml_parser_fetch_flow_collection_end(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset any potential simple key on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Decrease the flow level. + if !yaml_parser_decrease_flow_level(parser) { + return false + } + + // No simple keys after the indicators ']' and '}'. + parser.simple_key_allowed = false + + // Consume the token. + + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-ENTRY token. +func yaml_parser_fetch_flow_entry(parser *yaml_parser_t) bool { + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after ','. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_FLOW_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the BLOCK-ENTRY token. +func yaml_parser_fetch_block_entry(parser *yaml_parser_t) bool { + // Check if the scanner is in the block context. + if parser.flow_level == 0 { + // Check if we are allowed to start a new entry. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "block sequence entries are not allowed in this context") + } + // Add the BLOCK-SEQUENCE-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_SEQUENCE_START_TOKEN, parser.mark) { + return false + } + } else { + // It is an error for the '-' indicator to occur in the flow context, + // but we let the Parser detect and report about it because the Parser + // is able to point to the context. + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '-'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the BLOCK-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the KEY token. +func yaml_parser_fetch_key(parser *yaml_parser_t) bool { + + // In the block context, additional checks are required. + if parser.flow_level == 0 { + // Check if we are allowed to start a new key (not nessesary simple). + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping keys are not allowed in this context") + } + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '?' in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the KEY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the VALUE token. +func yaml_parser_fetch_value(parser *yaml_parser_t) bool { + + simple_key := &parser.simple_keys[len(parser.simple_keys)-1] + + // Have we found a simple key? + if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok { + return false + + } else if valid { + + // Create the KEY token and insert it into the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: simple_key.mark, + end_mark: simple_key.mark, + } + yaml_insert_token(parser, simple_key.token_number-parser.tokens_parsed, &token) + + // In the block context, we may need to add the BLOCK-MAPPING-START token. + if !yaml_parser_roll_indent(parser, simple_key.mark.column, + simple_key.token_number, + yaml_BLOCK_MAPPING_START_TOKEN, simple_key.mark) { + return false + } + + // Remove the simple key. + simple_key.possible = false + delete(parser.simple_keys_by_tok, simple_key.token_number) + + // A simple key cannot follow another simple key. + parser.simple_key_allowed = false + + } else { + // The ':' indicator follows a complex key. + + // In the block context, extra checks are required. + if parser.flow_level == 0 { + + // Check if we are allowed to start a complex value. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping values are not allowed in this context") + } + + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Simple keys after ':' are allowed in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + } + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the VALUE token and append it to the queue. + token := yaml_token_t{ + typ: yaml_VALUE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the ALIAS or ANCHOR token. +func yaml_parser_fetch_anchor(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // An anchor or an alias could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow an anchor or an alias. + parser.simple_key_allowed = false + + // Create the ALIAS or ANCHOR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_anchor(parser, &token, typ) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the TAG token. +func yaml_parser_fetch_tag(parser *yaml_parser_t) bool { + // A tag could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a tag. + parser.simple_key_allowed = false + + // Create the TAG token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_tag(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens. +func yaml_parser_fetch_block_scalar(parser *yaml_parser_t, literal bool) bool { + // Remove any potential simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // A simple key may follow a block scalar. + parser.simple_key_allowed = true + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_block_scalar(parser, &token, literal) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens. +func yaml_parser_fetch_flow_scalar(parser *yaml_parser_t, single bool) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_flow_scalar(parser, &token, single) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,plain) token. +func yaml_parser_fetch_plain_scalar(parser *yaml_parser_t) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_plain_scalar(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Eat whitespaces and comments until the next token is found. +func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { + + scan_mark := parser.mark + + // Until the next token is not found. + for { + // Allow the BOM mark to start a line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) { + skip(parser) + } + + // Eat whitespaces. + // Tabs are allowed: + // - in the flow context + // - in the block context, but not at the beginning of the line or + // after '-', '?', or ':' (complex value). + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if we just had a line comment under a sequence entry that + // looks more like a header to the following content. Similar to this: + // + // - # The comment + // - Some data + // + // If so, transform the line comment to a head comment and reposition. + if len(parser.comments) > 0 && len(parser.tokens) > 1 { + tokenA := parser.tokens[len(parser.tokens)-2] + tokenB := parser.tokens[len(parser.tokens)-1] + comment := &parser.comments[len(parser.comments)-1] + if tokenA.typ == yaml_BLOCK_SEQUENCE_START_TOKEN && tokenB.typ == yaml_BLOCK_ENTRY_TOKEN && len(comment.line) > 0 && !is_break(parser.buffer, parser.buffer_pos) { + // If it was in the prior line, reposition so it becomes a + // header of the follow up token. Otherwise, keep it in place + // so it becomes a header of the former. + comment.head = comment.line + comment.line = nil + if comment.start_mark.line == parser.mark.line-1 { + comment.token_mark = parser.mark + } + } + } + + // Eat a comment until a line break. + if parser.buffer[parser.buffer_pos] == '#' { + if !yaml_parser_scan_comments(parser, scan_mark) { + return false + } + } + + // If it is a line break, eat it. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + + // In the block context, a new line may start a simple key. + if parser.flow_level == 0 { + parser.simple_key_allowed = true + } + } else { + break // We have found a token. + } + } + + return true +} + +// Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool { + // Eat '%'. + start_mark := parser.mark + skip(parser) + + // Scan the directive name. + var name []byte + if !yaml_parser_scan_directive_name(parser, start_mark, &name) { + return false + } + + // Is it a YAML directive? + if bytes.Equal(name, []byte("YAML")) { + // Scan the VERSION directive value. + var major, minor int8 + if !yaml_parser_scan_version_directive_value(parser, start_mark, &major, &minor) { + return false + } + end_mark := parser.mark + + // Create a VERSION-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_VERSION_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + major: major, + minor: minor, + } + + // Is it a TAG directive? + } else if bytes.Equal(name, []byte("TAG")) { + // Scan the TAG directive value. + var handle, prefix []byte + if !yaml_parser_scan_tag_directive_value(parser, start_mark, &handle, &prefix) { + return false + } + end_mark := parser.mark + + // Create a TAG-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_TAG_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + prefix: prefix, + } + + // Unknown directive. + } else { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unknown directive name") + return false + } + + // Eat the rest of the line including any comments. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + if parser.buffer[parser.buffer_pos] == '#' { + // [Go] Discard this inline comment for the time being. + //if !yaml_parser_scan_line_comment(parser, start_mark) { + // return false + //} + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + return true +} + +// Scan the directive name. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^ +// +func yaml_parser_scan_directive_name(parser *yaml_parser_t, start_mark yaml_mark_t, name *[]byte) bool { + // Consume the directive name. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + var s []byte + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the name is empty. + if len(s) == 0 { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "could not find expected directive name") + return false + } + + // Check for an blank character after the name. + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unexpected non-alphabetical character") + return false + } + *name = s + return true +} + +// Scan the value of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^ +func yaml_parser_scan_version_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, major, minor *int8) bool { + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the major version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, major) { + return false + } + + // Eat '.'. + if parser.buffer[parser.buffer_pos] != '.' { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected digit or '.' character") + } + + skip(parser) + + // Consume the minor version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, minor) { + return false + } + return true +} + +const max_number_length = 2 + +// Scan the version number of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^ +// %YAML 1.1 # a comment \n +// ^ +func yaml_parser_scan_version_directive_number(parser *yaml_parser_t, start_mark yaml_mark_t, number *int8) bool { + + // Repeat while the next character is digit. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var value, length int8 + for is_digit(parser.buffer, parser.buffer_pos) { + // Check if the number is too long. + length++ + if length > max_number_length { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "found extremely long version number") + } + value = value*10 + int8(as_digit(parser.buffer, parser.buffer_pos)) + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the number was present. + if length == 0 { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected version number") + } + *number = value + return true +} + +// Scan the value of a TAG-DIRECTIVE token. +// +// Scope: +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_tag_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, handle, prefix *[]byte) bool { + var handle_value, prefix_value []byte + + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a handle. + if !yaml_parser_scan_tag_handle(parser, true, start_mark, &handle_value) { + return false + } + + // Expect a whitespace. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blank(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace") + return false + } + + // Eat whitespaces. + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a prefix. + if !yaml_parser_scan_tag_uri(parser, true, nil, start_mark, &prefix_value) { + return false + } + + // Expect a whitespace or line break. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace or line break") + return false + } + + *handle = handle_value + *prefix = prefix_value + return true +} + +func yaml_parser_scan_anchor(parser *yaml_parser_t, token *yaml_token_t, typ yaml_token_type_t) bool { + var s []byte + + // Eat the indicator character. + start_mark := parser.mark + skip(parser) + + // Consume the value. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + end_mark := parser.mark + + /* + * Check if length of the anchor is greater than 0 and it is followed by + * a whitespace character or one of the indicators: + * + * '?', ':', ',', ']', '}', '%', '@', '`'. + */ + + if len(s) == 0 || + !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '}' || + parser.buffer[parser.buffer_pos] == '%' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '`') { + context := "while scanning an alias" + if typ == yaml_ANCHOR_TOKEN { + context = "while scanning an anchor" + } + yaml_parser_set_scanner_error(parser, context, start_mark, + "did not find expected alphabetic or numeric character") + return false + } + + // Create a token. + *token = yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + value: s, + } + + return true +} + +/* + * Scan a TAG token. + */ + +func yaml_parser_scan_tag(parser *yaml_parser_t, token *yaml_token_t) bool { + var handle, suffix []byte + + start_mark := parser.mark + + // Check if the tag is in the canonical form. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + if parser.buffer[parser.buffer_pos+1] == '<' { + // Keep the handle as '' + + // Eat '!<' + skip(parser) + skip(parser) + + // Consume the tag value. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + + // Check for '>' and eat it. + if parser.buffer[parser.buffer_pos] != '>' { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find the expected '>'") + return false + } + + skip(parser) + } else { + // The tag has either the '!suffix' or the '!handle!suffix' form. + + // First, try to scan a handle. + if !yaml_parser_scan_tag_handle(parser, false, start_mark, &handle) { + return false + } + + // Check if it is, indeed, handle. + if handle[0] == '!' && len(handle) > 1 && handle[len(handle)-1] == '!' { + // Scan the suffix now. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + } else { + // It wasn't a handle after all. Scan the rest of the tag. + if !yaml_parser_scan_tag_uri(parser, false, handle, start_mark, &suffix) { + return false + } + + // Set the handle to '!'. + handle = []byte{'!'} + + // A special case: the '!' tag. Set the handle to '' and the + // suffix to '!'. + if len(suffix) == 0 { + handle, suffix = suffix, handle + } + } + } + + // Check the character which ends the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find expected whitespace or line break") + return false + } + + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_TAG_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + suffix: suffix, + } + return true +} + +// Scan a tag handle. +func yaml_parser_scan_tag_handle(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, handle *[]byte) bool { + // Check the initial '!' character. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] != '!' { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + + var s []byte + + // Copy the '!' character. + s = read(parser, s) + + // Copy all subsequent alphabetical and numerical characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the trailing character is '!' and copy it. + if parser.buffer[parser.buffer_pos] == '!' { + s = read(parser, s) + } else { + // It's either the '!' tag or not really a tag handle. If it's a %TAG + // directive, it's an error. If it's a tag token, it must be a part of URI. + if directive && string(s) != "!" { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + } + + *handle = s + return true +} + +// Scan a tag. +func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte, start_mark yaml_mark_t, uri *[]byte) bool { + //size_t length = head ? strlen((char *)head) : 0 + var s []byte + hasTag := len(head) > 0 + + // Copy the head if needed. + // + // Note that we don't copy the leading '!' character. + if len(head) > 1 { + s = append(s, head[1:]...) + } + + // Scan the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // The set of characters that may appear in URI is as follows: + // + // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&', + // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']', + // '%'. + // [Go] TODO Convert this into more reasonable logic. + for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' || + parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '=' || + parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '$' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '.' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '~' || + parser.buffer[parser.buffer_pos] == '*' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '(' || parser.buffer[parser.buffer_pos] == ')' || + parser.buffer[parser.buffer_pos] == '[' || parser.buffer[parser.buffer_pos] == ']' || + parser.buffer[parser.buffer_pos] == '%' { + // Check if it is a URI-escape sequence. + if parser.buffer[parser.buffer_pos] == '%' { + if !yaml_parser_scan_uri_escapes(parser, directive, start_mark, &s) { + return false + } + } else { + s = read(parser, s) + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + hasTag = true + } + + if !hasTag { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected tag URI") + return false + } + *uri = s + return true +} + +// Decode an URI-escape sequence corresponding to a single UTF-8 character. +func yaml_parser_scan_uri_escapes(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, s *[]byte) bool { + + // Decode the required number of characters. + w := 1024 + for w > 0 { + // Check for a URI-escaped octet. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + + if !(parser.buffer[parser.buffer_pos] == '%' && + is_hex(parser.buffer, parser.buffer_pos+1) && + is_hex(parser.buffer, parser.buffer_pos+2)) { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find URI escaped octet") + } + + // Get the octet. + octet := byte((as_hex(parser.buffer, parser.buffer_pos+1) << 4) + as_hex(parser.buffer, parser.buffer_pos+2)) + + // If it is the leading octet, determine the length of the UTF-8 sequence. + if w == 1024 { + w = width(octet) + if w == 0 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect leading UTF-8 octet") + } + } else { + // Check if the trailing octet is correct. + if octet&0xC0 != 0x80 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect trailing UTF-8 octet") + } + } + + // Copy the octet and move the pointers. + *s = append(*s, octet) + skip(parser) + skip(parser) + skip(parser) + w-- + } + return true +} + +// Scan a block scalar. +func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, literal bool) bool { + // Eat the indicator '|' or '>'. + start_mark := parser.mark + skip(parser) + + // Scan the additional block scalar indicators. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check for a chomping indicator. + var chomping, increment int + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + // Set the chomping method and eat the indicator. + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + + // Check for an indentation indicator. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_digit(parser.buffer, parser.buffer_pos) { + // Check that the indentation is greater than 0. + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + + // Get the indentation level and eat the indicator. + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + } + + } else if is_digit(parser.buffer, parser.buffer_pos) { + // Do the same as above, but in the opposite order. + + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + } + } + + // Eat whitespaces and comments to the end of the line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.buffer[parser.buffer_pos] == '#' { + if !yaml_parser_scan_line_comment(parser, start_mark) { + return false + } + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + end_mark := parser.mark + + // Set the indentation level if it was specified. + var indent int + if increment > 0 { + if parser.indent >= 0 { + indent = parser.indent + increment + } else { + indent = increment + } + } + + // Scan the leading line breaks and determine the indentation level if needed. + var s, leading_break, trailing_breaks []byte + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + + // Scan the block scalar content. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var leading_blank, trailing_blank bool + for parser.mark.column == indent && !is_z(parser.buffer, parser.buffer_pos) { + // We are at the beginning of a non-empty line. + + // Is it a trailing whitespace? + trailing_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Check if we need to fold the leading line break. + if !literal && !leading_blank && !trailing_blank && len(leading_break) > 0 && leading_break[0] == '\n' { + // Do we need to join the lines by space? + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } + } else { + s = append(s, leading_break...) + } + leading_break = leading_break[:0] + + // Append the remaining line breaks. + s = append(s, trailing_breaks...) + trailing_breaks = trailing_breaks[:0] + + // Is it a leading whitespace? + leading_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Consume the current line. + for !is_breakz(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + leading_break = read_line(parser, leading_break) + + // Eat the following indentation spaces and line breaks. + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + } + + // Chomp the tail. + if chomping != -1 { + s = append(s, leading_break...) + } + if chomping == 1 { + s = append(s, trailing_breaks...) + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_LITERAL_SCALAR_STYLE, + } + if !literal { + token.style = yaml_FOLDED_SCALAR_STYLE + } + return true +} + +// Scan indentation spaces and line breaks for a block scalar. Determine the +// indentation level if needed. +func yaml_parser_scan_block_scalar_breaks(parser *yaml_parser_t, indent *int, breaks *[]byte, start_mark yaml_mark_t, end_mark *yaml_mark_t) bool { + *end_mark = parser.mark + + // Eat the indentation spaces and line breaks. + max_indent := 0 + for { + // Eat the indentation spaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for (*indent == 0 || parser.mark.column < *indent) && is_space(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.mark.column > max_indent { + max_indent = parser.mark.column + } + + // Check for a tab character messing the indentation. + if (*indent == 0 || parser.mark.column < *indent) && is_tab(parser.buffer, parser.buffer_pos) { + return yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found a tab character where an indentation space is expected") + } + + // Have we found a non-empty line? + if !is_break(parser.buffer, parser.buffer_pos) { + break + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + // [Go] Should really be returning breaks instead. + *breaks = read_line(parser, *breaks) + *end_mark = parser.mark + } + + // Determine the indentation level if needed. + if *indent == 0 { + *indent = max_indent + if *indent < parser.indent+1 { + *indent = parser.indent + 1 + } + if *indent < 1 { + *indent = 1 + } + } + return true +} + +// Scan a quoted scalar. +func yaml_parser_scan_flow_scalar(parser *yaml_parser_t, token *yaml_token_t, single bool) bool { + // Eat the left quote. + start_mark := parser.mark + skip(parser) + + // Consume the content of the quoted scalar. + var s, leading_break, trailing_breaks, whitespaces []byte + for { + // Check that there are no document indicators at the beginning of the line. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected document indicator") + return false + } + + // Check for EOF. + if is_z(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected end of stream") + return false + } + + // Consume non-blank characters. + leading_blanks := false + for !is_blankz(parser.buffer, parser.buffer_pos) { + if single && parser.buffer[parser.buffer_pos] == '\'' && parser.buffer[parser.buffer_pos+1] == '\'' { + // Is is an escaped single quote. + s = append(s, '\'') + skip(parser) + skip(parser) + + } else if single && parser.buffer[parser.buffer_pos] == '\'' { + // It is a right single quote. + break + } else if !single && parser.buffer[parser.buffer_pos] == '"' { + // It is a right double quote. + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' && is_break(parser.buffer, parser.buffer_pos+1) { + // It is an escaped line break. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + skip(parser) + skip_line(parser) + leading_blanks = true + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' { + // It is an escape sequence. + code_length := 0 + + // Check the escape character. + switch parser.buffer[parser.buffer_pos+1] { + case '0': + s = append(s, 0) + case 'a': + s = append(s, '\x07') + case 'b': + s = append(s, '\x08') + case 't', '\t': + s = append(s, '\x09') + case 'n': + s = append(s, '\x0A') + case 'v': + s = append(s, '\x0B') + case 'f': + s = append(s, '\x0C') + case 'r': + s = append(s, '\x0D') + case 'e': + s = append(s, '\x1B') + case ' ': + s = append(s, '\x20') + case '"': + s = append(s, '"') + case '\'': + s = append(s, '\'') + case '\\': + s = append(s, '\\') + case 'N': // NEL (#x85) + s = append(s, '\xC2') + s = append(s, '\x85') + case '_': // #xA0 + s = append(s, '\xC2') + s = append(s, '\xA0') + case 'L': // LS (#x2028) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA8') + case 'P': // PS (#x2029) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA9') + case 'x': + code_length = 2 + case 'u': + code_length = 4 + case 'U': + code_length = 8 + default: + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found unknown escape character") + return false + } + + skip(parser) + skip(parser) + + // Consume an arbitrary escape code. + if code_length > 0 { + var value int + + // Scan the character value. + if parser.unread < code_length && !yaml_parser_update_buffer(parser, code_length) { + return false + } + for k := 0; k < code_length; k++ { + if !is_hex(parser.buffer, parser.buffer_pos+k) { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "did not find expected hexdecimal number") + return false + } + value = (value << 4) + as_hex(parser.buffer, parser.buffer_pos+k) + } + + // Check the value and write the character. + if (value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found invalid Unicode character escape code") + return false + } + if value <= 0x7F { + s = append(s, byte(value)) + } else if value <= 0x7FF { + s = append(s, byte(0xC0+(value>>6))) + s = append(s, byte(0x80+(value&0x3F))) + } else if value <= 0xFFFF { + s = append(s, byte(0xE0+(value>>12))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } else { + s = append(s, byte(0xF0+(value>>18))) + s = append(s, byte(0x80+((value>>12)&0x3F))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } + + // Advance the pointer. + for k := 0; k < code_length; k++ { + skip(parser) + } + } + } else { + // It is a non-escaped non-blank character. + s = read(parser, s) + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we are at the end of the scalar. + if single { + if parser.buffer[parser.buffer_pos] == '\'' { + break + } + } else { + if parser.buffer[parser.buffer_pos] == '"' { + break + } + } + + // Consume blank characters. + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Join the whitespaces or fold line breaks. + if leading_blanks { + // Do we need to fold line breaks? + if len(leading_break) > 0 && leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Eat the right quote. + skip(parser) + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_SINGLE_QUOTED_SCALAR_STYLE, + } + if !single { + token.style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + return true +} + +// Scan a plain scalar. +func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) bool { + + var s, leading_break, trailing_breaks, whitespaces []byte + var leading_blanks bool + var indent = parser.indent + 1 + + start_mark := parser.mark + end_mark := parser.mark + + // Consume the content of the plain scalar. + for { + // Check for a document indicator. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + break + } + + // Check for a comment. + if parser.buffer[parser.buffer_pos] == '#' { + break + } + + // Consume non-blank characters. + for !is_blankz(parser.buffer, parser.buffer_pos) { + + // Check for indicators that may end a plain scalar. + if (parser.buffer[parser.buffer_pos] == ':' && is_blankz(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level > 0 && + (parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}')) { + break + } + + // Check if we need to join whitespaces and breaks. + if leading_blanks || len(whitespaces) > 0 { + if leading_blanks { + // Do we need to fold line breaks? + if leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + leading_blanks = false + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Copy the character. + s = read(parser, s) + + end_mark = parser.mark + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + // Is it the end? + if !(is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos)) { + break + } + + // Consume blank characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + + // Check for tab characters that abuse indentation. + if leading_blanks && parser.mark.column < indent && is_tab(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a plain scalar", + start_mark, "found a tab character that violates indentation") + return false + } + + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check indentation level. + if parser.flow_level == 0 && parser.mark.column < indent { + break + } + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_PLAIN_SCALAR_STYLE, + } + + // Note that we change the 'simple_key_allowed' flag. + if leading_blanks { + parser.simple_key_allowed = true + } + return true +} + +func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t) bool { + if parser.newlines > 0 { + return true + } + + var start_mark yaml_mark_t + var text []byte + + for peek := 0; peek < 512; peek++ { + if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) { + break + } + if is_blank(parser.buffer, parser.buffer_pos+peek) { + continue + } + if parser.buffer[parser.buffer_pos+peek] == '#' { + seen := parser.mark.index+peek + for { + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_breakz(parser.buffer, parser.buffer_pos) { + if parser.mark.index >= seen { + break + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } else if parser.mark.index >= seen { + if len(text) == 0 { + start_mark = parser.mark + } + text = read(parser, text) + } else { + skip(parser) + } + } + } + break + } + if len(text) > 0 { + parser.comments = append(parser.comments, yaml_comment_t{ + token_mark: token_mark, + start_mark: start_mark, + line: text, + }) + } + return true +} + +func yaml_parser_scan_comments(parser *yaml_parser_t, scan_mark yaml_mark_t) bool { + token := parser.tokens[len(parser.tokens)-1] + + if token.typ == yaml_FLOW_ENTRY_TOKEN && len(parser.tokens) > 1 { + token = parser.tokens[len(parser.tokens)-2] + } + + var token_mark = token.start_mark + var start_mark yaml_mark_t + var next_indent = parser.indent + if next_indent < 0 { + next_indent = 0 + } + + var recent_empty = false + var first_empty = parser.newlines <= 1 + + var line = parser.mark.line + var column = parser.mark.column + + var text []byte + + // The foot line is the place where a comment must start to + // still be considered as a foot of the prior content. + // If there's some content in the currently parsed line, then + // the foot is the line below it. + var foot_line = -1 + if scan_mark.line > 0 { + foot_line = parser.mark.line-parser.newlines+1 + if parser.newlines == 0 && parser.mark.column > 1 { + foot_line++ + } + } + + var peek = 0 + for ; peek < 512; peek++ { + if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) { + break + } + column++ + if is_blank(parser.buffer, parser.buffer_pos+peek) { + continue + } + c := parser.buffer[parser.buffer_pos+peek] + var close_flow = parser.flow_level > 0 && (c == ']' || c == '}') + if close_flow || is_breakz(parser.buffer, parser.buffer_pos+peek) { + // Got line break or terminator. + if close_flow || !recent_empty { + if close_flow || first_empty && (start_mark.line == foot_line && token.typ != yaml_VALUE_TOKEN || start_mark.column-1 < next_indent) { + // This is the first empty line and there were no empty lines before, + // so this initial part of the comment is a foot of the prior token + // instead of being a head for the following one. Split it up. + // Alternatively, this might also be the last comment inside a flow + // scope, so it must be a footer. + if len(text) > 0 { + if start_mark.column-1 < next_indent { + // If dedented it's unrelated to the prior token. + token_mark = start_mark + } + parser.comments = append(parser.comments, yaml_comment_t{ + scan_mark: scan_mark, + token_mark: token_mark, + start_mark: start_mark, + end_mark: yaml_mark_t{parser.mark.index + peek, line, column}, + foot: text, + }) + scan_mark = yaml_mark_t{parser.mark.index + peek, line, column} + token_mark = scan_mark + text = nil + } + } else { + if len(text) > 0 && parser.buffer[parser.buffer_pos+peek] != 0 { + text = append(text, '\n') + } + } + } + if !is_break(parser.buffer, parser.buffer_pos+peek) { + break + } + first_empty = false + recent_empty = true + column = 0 + line++ + continue + } + + if len(text) > 0 && (close_flow || column-1 < next_indent && column != start_mark.column) { + // The comment at the different indentation is a foot of the + // preceding data rather than a head of the upcoming one. + parser.comments = append(parser.comments, yaml_comment_t{ + scan_mark: scan_mark, + token_mark: token_mark, + start_mark: start_mark, + end_mark: yaml_mark_t{parser.mark.index + peek, line, column}, + foot: text, + }) + scan_mark = yaml_mark_t{parser.mark.index + peek, line, column} + token_mark = scan_mark + text = nil + } + + if parser.buffer[parser.buffer_pos+peek] != '#' { + break + } + + if len(text) == 0 { + start_mark = yaml_mark_t{parser.mark.index + peek, line, column} + } else { + text = append(text, '\n') + } + + recent_empty = false + + // Consume until after the consumed comment line. + seen := parser.mark.index+peek + for { + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_breakz(parser.buffer, parser.buffer_pos) { + if parser.mark.index >= seen { + break + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } else if parser.mark.index >= seen { + text = read(parser, text) + } else { + skip(parser) + } + } + + peek = 0 + column = 0 + line = parser.mark.line + next_indent = parser.indent + if next_indent < 0 { + next_indent = 0 + } + } + + if len(text) > 0 { + parser.comments = append(parser.comments, yaml_comment_t{ + scan_mark: scan_mark, + token_mark: start_mark, + start_mark: start_mark, + end_mark: yaml_mark_t{parser.mark.index + peek - 1, line, column}, + head: text, + }) + } + return true +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/sorter.go b/scraper-go/vendor/gopkg.in/yaml.v3/sorter.go new file mode 100644 index 00000000..9210ece7 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/sorter.go @@ -0,0 +1,134 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "reflect" + "unicode" +) + +type keyList []reflect.Value + +func (l keyList) Len() int { return len(l) } +func (l keyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l keyList) Less(i, j int) bool { + a := l[i] + b := l[j] + ak := a.Kind() + bk := b.Kind() + for (ak == reflect.Interface || ak == reflect.Ptr) && !a.IsNil() { + a = a.Elem() + ak = a.Kind() + } + for (bk == reflect.Interface || bk == reflect.Ptr) && !b.IsNil() { + b = b.Elem() + bk = b.Kind() + } + af, aok := keyFloat(a) + bf, bok := keyFloat(b) + if aok && bok { + if af != bf { + return af < bf + } + if ak != bk { + return ak < bk + } + return numLess(a, b) + } + if ak != reflect.String || bk != reflect.String { + return ak < bk + } + ar, br := []rune(a.String()), []rune(b.String()) + digits := false + for i := 0; i < len(ar) && i < len(br); i++ { + if ar[i] == br[i] { + digits = unicode.IsDigit(ar[i]) + continue + } + al := unicode.IsLetter(ar[i]) + bl := unicode.IsLetter(br[i]) + if al && bl { + return ar[i] < br[i] + } + if al || bl { + if digits { + return al + } else { + return bl + } + } + var ai, bi int + var an, bn int64 + if ar[i] == '0' || br[i] == '0' { + for j := i - 1; j >= 0 && unicode.IsDigit(ar[j]); j-- { + if ar[j] != '0' { + an = 1 + bn = 1 + break + } + } + } + for ai = i; ai < len(ar) && unicode.IsDigit(ar[ai]); ai++ { + an = an*10 + int64(ar[ai]-'0') + } + for bi = i; bi < len(br) && unicode.IsDigit(br[bi]); bi++ { + bn = bn*10 + int64(br[bi]-'0') + } + if an != bn { + return an < bn + } + if ai != bi { + return ai < bi + } + return ar[i] < br[i] + } + return len(ar) < len(br) +} + +// keyFloat returns a float value for v if it is a number/bool +// and whether it is a number/bool or not. +func keyFloat(v reflect.Value) (f float64, ok bool) { + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(v.Int()), true + case reflect.Float32, reflect.Float64: + return v.Float(), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return float64(v.Uint()), true + case reflect.Bool: + if v.Bool() { + return 1, true + } + return 0, true + } + return 0, false +} + +// numLess returns whether a < b. +// a and b must necessarily have the same kind. +func numLess(a, b reflect.Value) bool { + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return a.Uint() < b.Uint() + case reflect.Bool: + return !a.Bool() && b.Bool() + } + panic("not a number") +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/writerc.go b/scraper-go/vendor/gopkg.in/yaml.v3/writerc.go new file mode 100644 index 00000000..b8a116bf --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/writerc.go @@ -0,0 +1,48 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +// Set the writer error and return false. +func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_WRITER_ERROR + emitter.problem = problem + return false +} + +// Flush the output buffer. +func yaml_emitter_flush(emitter *yaml_emitter_t) bool { + if emitter.write_handler == nil { + panic("write handler not set") + } + + // Check if the buffer is empty. + if emitter.buffer_pos == 0 { + return true + } + + if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil { + return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error()) + } + emitter.buffer_pos = 0 + return true +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/yaml.go b/scraper-go/vendor/gopkg.in/yaml.v3/yaml.go new file mode 100644 index 00000000..8cec6da4 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/yaml.go @@ -0,0 +1,698 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package yaml implements YAML support for the Go language. +// +// Source code and other details for the project are available at GitHub: +// +// https://github.com/go-yaml/yaml +// +package yaml + +import ( + "errors" + "fmt" + "io" + "reflect" + "strings" + "sync" + "unicode/utf8" +) + +// The Unmarshaler interface may be implemented by types to customize their +// behavior when being unmarshaled from a YAML document. +type Unmarshaler interface { + UnmarshalYAML(value *Node) error +} + +type obsoleteUnmarshaler interface { + UnmarshalYAML(unmarshal func(interface{}) error) error +} + +// The Marshaler interface may be implemented by types to customize their +// behavior when being marshaled into a YAML document. The returned value +// is marshaled in place of the original value implementing Marshaler. +// +// If an error is returned by MarshalYAML, the marshaling procedure stops +// and returns with the provided error. +type Marshaler interface { + MarshalYAML() (interface{}, error) +} + +// Unmarshal decodes the first document found within the in byte slice +// and assigns decoded values into the out value. +// +// Maps and pointers (to a struct, string, int, etc) are accepted as out +// values. If an internal pointer within a struct is not initialized, +// the yaml package will initialize it if necessary for unmarshalling +// the provided data. The out parameter must not be nil. +// +// The type of the decoded values should be compatible with the respective +// values in out. If one or more values cannot be decoded due to a type +// mismatches, decoding continues partially until the end of the YAML +// content, and a *yaml.TypeError is returned with details for all +// missed values. +// +// Struct fields are only unmarshalled if they are exported (have an +// upper case first letter), and are unmarshalled using the field name +// lowercased as the default key. Custom keys may be defined via the +// "yaml" name in the field tag: the content preceding the first comma +// is used as the key, and the following comma-separated options are +// used to tweak the marshalling process (see Marshal). +// Conflicting names result in a runtime error. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// +// See the documentation of Marshal for the format of tags and a list of +// supported tag options. +// +func Unmarshal(in []byte, out interface{}) (err error) { + return unmarshal(in, out, false) +} + +// A Decoder reads and decodes YAML values from an input stream. +type Decoder struct { + parser *parser + knownFields bool +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read +// data from r beyond the YAML values requested. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + parser: newParserFromReader(r), + } +} + +// KnownFields ensures that the keys in decoded mappings to +// exist as fields in the struct being decoded into. +func (dec *Decoder) KnownFields(enable bool) { + dec.knownFields = enable +} + +// Decode reads the next YAML-encoded value from its input +// and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (dec *Decoder) Decode(v interface{}) (err error) { + d := newDecoder() + d.knownFields = dec.knownFields + defer handleErr(&err) + node := dec.parser.parse() + if node == nil { + return io.EOF + } + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(node, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Decode decodes the node and stores its data into the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (n *Node) Decode(v interface{}) (err error) { + d := newDecoder() + defer handleErr(&err) + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(n, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +func unmarshal(in []byte, out interface{}, strict bool) (err error) { + defer handleErr(&err) + d := newDecoder() + p := newParser(in) + defer p.destroy() + node := p.parse() + if node != nil { + v := reflect.ValueOf(out) + if v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + d.unmarshal(node, v) + } + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Marshal serializes the value provided into a YAML document. The structure +// of the generated document will reflect the structure of the value itself. +// Maps and pointers (to struct, string, int, etc) are accepted as the in value. +// +// Struct fields are only marshalled if they are exported (have an upper case +// first letter), and are marshalled using the field name lowercased as the +// default key. Custom keys may be defined via the "yaml" name in the field +// tag: the content preceding the first comma is used as the key, and the +// following comma-separated options are used to tweak the marshalling process. +// Conflicting names result in a runtime error. +// +// The field tag format accepted is: +// +// `(...) yaml:"[][,[,]]" (...)` +// +// The following flags are currently supported: +// +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be excluded if IsZero returns true. +// +// flow Marshal using a flow style (useful for structs, +// sequences and maps). +// +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. +// +// In addition, if the key is "-", the field is ignored. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" +// +func Marshal(in interface{}) (out []byte, err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(in)) + e.finish() + out = e.out + return +} + +// An Encoder writes YAML values to an output stream. +type Encoder struct { + encoder *encoder +} + +// NewEncoder returns a new encoder that writes to w. +// The Encoder should be closed after use to flush all data +// to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + encoder: newEncoderWithWriter(w), + } +} + +// Encode writes the YAML encoding of v to the stream. +// If multiple items are encoded to the stream, the +// second and subsequent document will be preceded +// with a "---" document separator, but the first will not. +// +// See the documentation for Marshal for details about the conversion of Go +// values to YAML. +func (e *Encoder) Encode(v interface{}) (err error) { + defer handleErr(&err) + e.encoder.marshalDoc("", reflect.ValueOf(v)) + return nil +} + +// Encode encodes value v and stores its representation in n. +// +// See the documentation for Marshal for details about the +// conversion of Go values into YAML. +func (n *Node) Encode(v interface{}) (err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(v)) + e.finish() + p := newParser(e.out) + p.textless = true + defer p.destroy() + doc := p.parse() + *n = *doc.Content[0] + return nil +} + +// SetIndent changes the used indentation used when encoding. +func (e *Encoder) SetIndent(spaces int) { + if spaces < 0 { + panic("yaml: cannot indent to a negative number of spaces") + } + e.encoder.indent = spaces +} + +// Close closes the encoder by writing any remaining data. +// It does not write a stream terminating string "...". +func (e *Encoder) Close() (err error) { + defer handleErr(&err) + e.encoder.finish() + return nil +} + +func handleErr(err *error) { + if v := recover(); v != nil { + if e, ok := v.(yamlError); ok { + *err = e.err + } else { + panic(v) + } + } +} + +type yamlError struct { + err error +} + +func fail(err error) { + panic(yamlError{err}) +} + +func failf(format string, args ...interface{}) { + panic(yamlError{fmt.Errorf("yaml: "+format, args...)}) +} + +// A TypeError is returned by Unmarshal when one or more fields in +// the YAML document cannot be properly decoded into the requested +// types. When this error is returned, the value is still +// unmarshaled partially. +type TypeError struct { + Errors []string +} + +func (e *TypeError) Error() string { + return fmt.Sprintf("yaml: unmarshal errors:\n %s", strings.Join(e.Errors, "\n ")) +} + +type Kind uint32 + +const ( + DocumentNode Kind = 1 << iota + SequenceNode + MappingNode + ScalarNode + AliasNode +) + +type Style uint32 + +const ( + TaggedStyle Style = 1 << iota + DoubleQuotedStyle + SingleQuotedStyle + LiteralStyle + FoldedStyle + FlowStyle +) + +// Node represents an element in the YAML document hierarchy. While documents +// are typically encoded and decoded into higher level types, such as structs +// and maps, Node is an intermediate representation that allows detailed +// control over the content being decoded or encoded. +// +// It's worth noting that although Node offers access into details such as +// line numbers, colums, and comments, the content when re-encoded will not +// have its original textual representation preserved. An effort is made to +// render the data plesantly, and to preserve comments near the data they +// describe, though. +// +// Values that make use of the Node type interact with the yaml package in the +// same way any other type would do, by encoding and decoding yaml data +// directly or indirectly into them. +// +// For example: +// +// var person struct { +// Name string +// Address yaml.Node +// } +// err := yaml.Unmarshal(data, &person) +// +// Or by itself: +// +// var person Node +// err := yaml.Unmarshal(data, &person) +// +type Node struct { + // Kind defines whether the node is a document, a mapping, a sequence, + // a scalar value, or an alias to another node. The specific data type of + // scalar nodes may be obtained via the ShortTag and LongTag methods. + Kind Kind + + // Style allows customizing the apperance of the node in the tree. + Style Style + + // Tag holds the YAML tag defining the data type for the value. + // When decoding, this field will always be set to the resolved tag, + // even when it wasn't explicitly provided in the YAML content. + // When encoding, if this field is unset the value type will be + // implied from the node properties, and if it is set, it will only + // be serialized into the representation if TaggedStyle is used or + // the implicit tag diverges from the provided one. + Tag string + + // Value holds the unescaped and unquoted represenation of the value. + Value string + + // Anchor holds the anchor name for this node, which allows aliases to point to it. + Anchor string + + // Alias holds the node that this alias points to. Only valid when Kind is AliasNode. + Alias *Node + + // Content holds contained nodes for documents, mappings, and sequences. + Content []*Node + + // HeadComment holds any comments in the lines preceding the node and + // not separated by an empty line. + HeadComment string + + // LineComment holds any comments at the end of the line where the node is in. + LineComment string + + // FootComment holds any comments following the node and before empty lines. + FootComment string + + // Line and Column hold the node position in the decoded YAML text. + // These fields are not respected when encoding the node. + Line int + Column int +} + +// IsZero returns whether the node has all of its fields unset. +func (n *Node) IsZero() bool { + return n.Kind == 0 && n.Style == 0 && n.Tag == "" && n.Value == "" && n.Anchor == "" && n.Alias == nil && n.Content == nil && + n.HeadComment == "" && n.LineComment == "" && n.FootComment == "" && n.Line == 0 && n.Column == 0 +} + + +// LongTag returns the long form of the tag that indicates the data type for +// the node. If the Tag field isn't explicitly defined, one will be computed +// based on the node properties. +func (n *Node) LongTag() string { + return longTag(n.ShortTag()) +} + +// ShortTag returns the short form of the YAML tag that indicates data type for +// the node. If the Tag field isn't explicitly defined, one will be computed +// based on the node properties. +func (n *Node) ShortTag() string { + if n.indicatedString() { + return strTag + } + if n.Tag == "" || n.Tag == "!" { + switch n.Kind { + case MappingNode: + return mapTag + case SequenceNode: + return seqTag + case AliasNode: + if n.Alias != nil { + return n.Alias.ShortTag() + } + case ScalarNode: + tag, _ := resolve("", n.Value) + return tag + case 0: + // Special case to make the zero value convenient. + if n.IsZero() { + return nullTag + } + } + return "" + } + return shortTag(n.Tag) +} + +func (n *Node) indicatedString() bool { + return n.Kind == ScalarNode && + (shortTag(n.Tag) == strTag || + (n.Tag == "" || n.Tag == "!") && n.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0) +} + +// SetString is a convenience function that sets the node to a string value +// and defines its style in a pleasant way depending on its content. +func (n *Node) SetString(s string) { + n.Kind = ScalarNode + if utf8.ValidString(s) { + n.Value = s + n.Tag = strTag + } else { + n.Value = encodeBase64(s) + n.Tag = binaryTag + } + if strings.Contains(n.Value, "\n") { + n.Style = LiteralStyle + } +} + +// -------------------------------------------------------------------------- +// Maintain a mapping of keys to structure field indexes + +// The code in this section was copied from mgo/bson. + +// structInfo holds details for the serialization of fields of +// a given struct. +type structInfo struct { + FieldsMap map[string]fieldInfo + FieldsList []fieldInfo + + // InlineMap is the number of the field in the struct that + // contains an ,inline map, or -1 if there's none. + InlineMap int + + // InlineUnmarshalers holds indexes to inlined fields that + // contain unmarshaler values. + InlineUnmarshalers [][]int +} + +type fieldInfo struct { + Key string + Num int + OmitEmpty bool + Flow bool + // Id holds the unique field identifier, so we can cheaply + // check for field duplicates without maintaining an extra map. + Id int + + // Inline holds the field index if the field is part of an inlined struct. + Inline []int +} + +var structMap = make(map[reflect.Type]*structInfo) +var fieldMapMutex sync.RWMutex +var unmarshalerType reflect.Type + +func init() { + var v Unmarshaler + unmarshalerType = reflect.ValueOf(&v).Elem().Type() +} + +func getStructInfo(st reflect.Type) (*structInfo, error) { + fieldMapMutex.RLock() + sinfo, found := structMap[st] + fieldMapMutex.RUnlock() + if found { + return sinfo, nil + } + + n := st.NumField() + fieldsMap := make(map[string]fieldInfo) + fieldsList := make([]fieldInfo, 0, n) + inlineMap := -1 + inlineUnmarshalers := [][]int(nil) + for i := 0; i != n; i++ { + field := st.Field(i) + if field.PkgPath != "" && !field.Anonymous { + continue // Private field + } + + info := fieldInfo{Num: i} + + tag := field.Tag.Get("yaml") + if tag == "" && strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + if tag == "-" { + continue + } + + inline := false + fields := strings.Split(tag, ",") + if len(fields) > 1 { + for _, flag := range fields[1:] { + switch flag { + case "omitempty": + info.OmitEmpty = true + case "flow": + info.Flow = true + case "inline": + inline = true + default: + return nil, errors.New(fmt.Sprintf("unsupported flag %q in tag %q of type %s", flag, tag, st)) + } + } + tag = fields[0] + } + + if inline { + switch field.Type.Kind() { + case reflect.Map: + if inlineMap >= 0 { + return nil, errors.New("multiple ,inline maps in struct " + st.String()) + } + if field.Type.Key() != reflect.TypeOf("") { + return nil, errors.New("option ,inline needs a map with string keys in struct " + st.String()) + } + inlineMap = info.Num + case reflect.Struct, reflect.Ptr: + ftype := field.Type + for ftype.Kind() == reflect.Ptr { + ftype = ftype.Elem() + } + if ftype.Kind() != reflect.Struct { + return nil, errors.New("option ,inline may only be used on a struct or map field") + } + if reflect.PtrTo(ftype).Implements(unmarshalerType) { + inlineUnmarshalers = append(inlineUnmarshalers, []int{i}) + } else { + sinfo, err := getStructInfo(ftype) + if err != nil { + return nil, err + } + for _, index := range sinfo.InlineUnmarshalers { + inlineUnmarshalers = append(inlineUnmarshalers, append([]int{i}, index...)) + } + for _, finfo := range sinfo.FieldsList { + if _, found := fieldsMap[finfo.Key]; found { + msg := "duplicated key '" + finfo.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + if finfo.Inline == nil { + finfo.Inline = []int{i, finfo.Num} + } else { + finfo.Inline = append([]int{i}, finfo.Inline...) + } + finfo.Id = len(fieldsList) + fieldsMap[finfo.Key] = finfo + fieldsList = append(fieldsList, finfo) + } + } + default: + return nil, errors.New("option ,inline may only be used on a struct or map field") + } + continue + } + + if tag != "" { + info.Key = tag + } else { + info.Key = strings.ToLower(field.Name) + } + + if _, found = fieldsMap[info.Key]; found { + msg := "duplicated key '" + info.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + + info.Id = len(fieldsList) + fieldsList = append(fieldsList, info) + fieldsMap[info.Key] = info + } + + sinfo = &structInfo{ + FieldsMap: fieldsMap, + FieldsList: fieldsList, + InlineMap: inlineMap, + InlineUnmarshalers: inlineUnmarshalers, + } + + fieldMapMutex.Lock() + structMap[st] = sinfo + fieldMapMutex.Unlock() + return sinfo, nil +} + +// IsZeroer is used to check whether an object is zero to +// determine whether it should be omitted when marshaling +// with the omitempty flag. One notable implementation +// is time.Time. +type IsZeroer interface { + IsZero() bool +} + +func isZero(v reflect.Value) bool { + kind := v.Kind() + if z, ok := v.Interface().(IsZeroer); ok { + if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() { + return true + } + return z.IsZero() + } + switch kind { + case reflect.String: + return len(v.String()) == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + case reflect.Slice: + return v.Len() == 0 + case reflect.Map: + return v.Len() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Struct: + vt := v.Type() + for i := v.NumField() - 1; i >= 0; i-- { + if vt.Field(i).PkgPath != "" { + continue // Private field + } + if !isZero(v.Field(i)) { + return false + } + } + return true + } + return false +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/yamlh.go b/scraper-go/vendor/gopkg.in/yaml.v3/yamlh.go new file mode 100644 index 00000000..7c6d0077 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/yamlh.go @@ -0,0 +1,807 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "fmt" + "io" +) + +// The version directive data. +type yaml_version_directive_t struct { + major int8 // The major version number. + minor int8 // The minor version number. +} + +// The tag directive data. +type yaml_tag_directive_t struct { + handle []byte // The tag handle. + prefix []byte // The tag prefix. +} + +type yaml_encoding_t int + +// The stream encoding. +const ( + // Let the parser choose the encoding. + yaml_ANY_ENCODING yaml_encoding_t = iota + + yaml_UTF8_ENCODING // The default UTF-8 encoding. + yaml_UTF16LE_ENCODING // The UTF-16-LE encoding with BOM. + yaml_UTF16BE_ENCODING // The UTF-16-BE encoding with BOM. +) + +type yaml_break_t int + +// Line break types. +const ( + // Let the parser choose the break type. + yaml_ANY_BREAK yaml_break_t = iota + + yaml_CR_BREAK // Use CR for line breaks (Mac style). + yaml_LN_BREAK // Use LN for line breaks (Unix style). + yaml_CRLN_BREAK // Use CR LN for line breaks (DOS style). +) + +type yaml_error_type_t int + +// Many bad things could happen with the parser and emitter. +const ( + // No error is produced. + yaml_NO_ERROR yaml_error_type_t = iota + + yaml_MEMORY_ERROR // Cannot allocate or reallocate a block of memory. + yaml_READER_ERROR // Cannot read or decode the input stream. + yaml_SCANNER_ERROR // Cannot scan the input stream. + yaml_PARSER_ERROR // Cannot parse the input stream. + yaml_COMPOSER_ERROR // Cannot compose a YAML document. + yaml_WRITER_ERROR // Cannot write to the output stream. + yaml_EMITTER_ERROR // Cannot emit a YAML stream. +) + +// The pointer position. +type yaml_mark_t struct { + index int // The position index. + line int // The position line. + column int // The position column. +} + +// Node Styles + +type yaml_style_t int8 + +type yaml_scalar_style_t yaml_style_t + +// Scalar styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = 0 + + yaml_PLAIN_SCALAR_STYLE yaml_scalar_style_t = 1 << iota // The plain scalar style. + yaml_SINGLE_QUOTED_SCALAR_STYLE // The single-quoted scalar style. + yaml_DOUBLE_QUOTED_SCALAR_STYLE // The double-quoted scalar style. + yaml_LITERAL_SCALAR_STYLE // The literal scalar style. + yaml_FOLDED_SCALAR_STYLE // The folded scalar style. +) + +type yaml_sequence_style_t yaml_style_t + +// Sequence styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SEQUENCE_STYLE yaml_sequence_style_t = iota + + yaml_BLOCK_SEQUENCE_STYLE // The block sequence style. + yaml_FLOW_SEQUENCE_STYLE // The flow sequence style. +) + +type yaml_mapping_style_t yaml_style_t + +// Mapping styles. +const ( + // Let the emitter choose the style. + yaml_ANY_MAPPING_STYLE yaml_mapping_style_t = iota + + yaml_BLOCK_MAPPING_STYLE // The block mapping style. + yaml_FLOW_MAPPING_STYLE // The flow mapping style. +) + +// Tokens + +type yaml_token_type_t int + +// Token types. +const ( + // An empty token. + yaml_NO_TOKEN yaml_token_type_t = iota + + yaml_STREAM_START_TOKEN // A STREAM-START token. + yaml_STREAM_END_TOKEN // A STREAM-END token. + + yaml_VERSION_DIRECTIVE_TOKEN // A VERSION-DIRECTIVE token. + yaml_TAG_DIRECTIVE_TOKEN // A TAG-DIRECTIVE token. + yaml_DOCUMENT_START_TOKEN // A DOCUMENT-START token. + yaml_DOCUMENT_END_TOKEN // A DOCUMENT-END token. + + yaml_BLOCK_SEQUENCE_START_TOKEN // A BLOCK-SEQUENCE-START token. + yaml_BLOCK_MAPPING_START_TOKEN // A BLOCK-SEQUENCE-END token. + yaml_BLOCK_END_TOKEN // A BLOCK-END token. + + yaml_FLOW_SEQUENCE_START_TOKEN // A FLOW-SEQUENCE-START token. + yaml_FLOW_SEQUENCE_END_TOKEN // A FLOW-SEQUENCE-END token. + yaml_FLOW_MAPPING_START_TOKEN // A FLOW-MAPPING-START token. + yaml_FLOW_MAPPING_END_TOKEN // A FLOW-MAPPING-END token. + + yaml_BLOCK_ENTRY_TOKEN // A BLOCK-ENTRY token. + yaml_FLOW_ENTRY_TOKEN // A FLOW-ENTRY token. + yaml_KEY_TOKEN // A KEY token. + yaml_VALUE_TOKEN // A VALUE token. + + yaml_ALIAS_TOKEN // An ALIAS token. + yaml_ANCHOR_TOKEN // An ANCHOR token. + yaml_TAG_TOKEN // A TAG token. + yaml_SCALAR_TOKEN // A SCALAR token. +) + +func (tt yaml_token_type_t) String() string { + switch tt { + case yaml_NO_TOKEN: + return "yaml_NO_TOKEN" + case yaml_STREAM_START_TOKEN: + return "yaml_STREAM_START_TOKEN" + case yaml_STREAM_END_TOKEN: + return "yaml_STREAM_END_TOKEN" + case yaml_VERSION_DIRECTIVE_TOKEN: + return "yaml_VERSION_DIRECTIVE_TOKEN" + case yaml_TAG_DIRECTIVE_TOKEN: + return "yaml_TAG_DIRECTIVE_TOKEN" + case yaml_DOCUMENT_START_TOKEN: + return "yaml_DOCUMENT_START_TOKEN" + case yaml_DOCUMENT_END_TOKEN: + return "yaml_DOCUMENT_END_TOKEN" + case yaml_BLOCK_SEQUENCE_START_TOKEN: + return "yaml_BLOCK_SEQUENCE_START_TOKEN" + case yaml_BLOCK_MAPPING_START_TOKEN: + return "yaml_BLOCK_MAPPING_START_TOKEN" + case yaml_BLOCK_END_TOKEN: + return "yaml_BLOCK_END_TOKEN" + case yaml_FLOW_SEQUENCE_START_TOKEN: + return "yaml_FLOW_SEQUENCE_START_TOKEN" + case yaml_FLOW_SEQUENCE_END_TOKEN: + return "yaml_FLOW_SEQUENCE_END_TOKEN" + case yaml_FLOW_MAPPING_START_TOKEN: + return "yaml_FLOW_MAPPING_START_TOKEN" + case yaml_FLOW_MAPPING_END_TOKEN: + return "yaml_FLOW_MAPPING_END_TOKEN" + case yaml_BLOCK_ENTRY_TOKEN: + return "yaml_BLOCK_ENTRY_TOKEN" + case yaml_FLOW_ENTRY_TOKEN: + return "yaml_FLOW_ENTRY_TOKEN" + case yaml_KEY_TOKEN: + return "yaml_KEY_TOKEN" + case yaml_VALUE_TOKEN: + return "yaml_VALUE_TOKEN" + case yaml_ALIAS_TOKEN: + return "yaml_ALIAS_TOKEN" + case yaml_ANCHOR_TOKEN: + return "yaml_ANCHOR_TOKEN" + case yaml_TAG_TOKEN: + return "yaml_TAG_TOKEN" + case yaml_SCALAR_TOKEN: + return "yaml_SCALAR_TOKEN" + } + return "" +} + +// The token structure. +type yaml_token_t struct { + // The token type. + typ yaml_token_type_t + + // The start/end of the token. + start_mark, end_mark yaml_mark_t + + // The stream encoding (for yaml_STREAM_START_TOKEN). + encoding yaml_encoding_t + + // The alias/anchor/scalar value or tag/tag directive handle + // (for yaml_ALIAS_TOKEN, yaml_ANCHOR_TOKEN, yaml_SCALAR_TOKEN, yaml_TAG_TOKEN, yaml_TAG_DIRECTIVE_TOKEN). + value []byte + + // The tag suffix (for yaml_TAG_TOKEN). + suffix []byte + + // The tag directive prefix (for yaml_TAG_DIRECTIVE_TOKEN). + prefix []byte + + // The scalar style (for yaml_SCALAR_TOKEN). + style yaml_scalar_style_t + + // The version directive major/minor (for yaml_VERSION_DIRECTIVE_TOKEN). + major, minor int8 +} + +// Events + +type yaml_event_type_t int8 + +// Event types. +const ( + // An empty event. + yaml_NO_EVENT yaml_event_type_t = iota + + yaml_STREAM_START_EVENT // A STREAM-START event. + yaml_STREAM_END_EVENT // A STREAM-END event. + yaml_DOCUMENT_START_EVENT // A DOCUMENT-START event. + yaml_DOCUMENT_END_EVENT // A DOCUMENT-END event. + yaml_ALIAS_EVENT // An ALIAS event. + yaml_SCALAR_EVENT // A SCALAR event. + yaml_SEQUENCE_START_EVENT // A SEQUENCE-START event. + yaml_SEQUENCE_END_EVENT // A SEQUENCE-END event. + yaml_MAPPING_START_EVENT // A MAPPING-START event. + yaml_MAPPING_END_EVENT // A MAPPING-END event. + yaml_TAIL_COMMENT_EVENT +) + +var eventStrings = []string{ + yaml_NO_EVENT: "none", + yaml_STREAM_START_EVENT: "stream start", + yaml_STREAM_END_EVENT: "stream end", + yaml_DOCUMENT_START_EVENT: "document start", + yaml_DOCUMENT_END_EVENT: "document end", + yaml_ALIAS_EVENT: "alias", + yaml_SCALAR_EVENT: "scalar", + yaml_SEQUENCE_START_EVENT: "sequence start", + yaml_SEQUENCE_END_EVENT: "sequence end", + yaml_MAPPING_START_EVENT: "mapping start", + yaml_MAPPING_END_EVENT: "mapping end", + yaml_TAIL_COMMENT_EVENT: "tail comment", +} + +func (e yaml_event_type_t) String() string { + if e < 0 || int(e) >= len(eventStrings) { + return fmt.Sprintf("unknown event %d", e) + } + return eventStrings[e] +} + +// The event structure. +type yaml_event_t struct { + + // The event type. + typ yaml_event_type_t + + // The start and end of the event. + start_mark, end_mark yaml_mark_t + + // The document encoding (for yaml_STREAM_START_EVENT). + encoding yaml_encoding_t + + // The version directive (for yaml_DOCUMENT_START_EVENT). + version_directive *yaml_version_directive_t + + // The list of tag directives (for yaml_DOCUMENT_START_EVENT). + tag_directives []yaml_tag_directive_t + + // The comments + head_comment []byte + line_comment []byte + foot_comment []byte + tail_comment []byte + + // The anchor (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_ALIAS_EVENT). + anchor []byte + + // The tag (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + tag []byte + + // The scalar value (for yaml_SCALAR_EVENT). + value []byte + + // Is the document start/end indicator implicit, or the tag optional? + // (for yaml_DOCUMENT_START_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_SCALAR_EVENT). + implicit bool + + // Is the tag optional for any non-plain style? (for yaml_SCALAR_EVENT). + quoted_implicit bool + + // The style (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + style yaml_style_t +} + +func (e *yaml_event_t) scalar_style() yaml_scalar_style_t { return yaml_scalar_style_t(e.style) } +func (e *yaml_event_t) sequence_style() yaml_sequence_style_t { return yaml_sequence_style_t(e.style) } +func (e *yaml_event_t) mapping_style() yaml_mapping_style_t { return yaml_mapping_style_t(e.style) } + +// Nodes + +const ( + yaml_NULL_TAG = "tag:yaml.org,2002:null" // The tag !!null with the only possible value: null. + yaml_BOOL_TAG = "tag:yaml.org,2002:bool" // The tag !!bool with the values: true and false. + yaml_STR_TAG = "tag:yaml.org,2002:str" // The tag !!str for string values. + yaml_INT_TAG = "tag:yaml.org,2002:int" // The tag !!int for integer values. + yaml_FLOAT_TAG = "tag:yaml.org,2002:float" // The tag !!float for float values. + yaml_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" // The tag !!timestamp for date and time values. + + yaml_SEQ_TAG = "tag:yaml.org,2002:seq" // The tag !!seq is used to denote sequences. + yaml_MAP_TAG = "tag:yaml.org,2002:map" // The tag !!map is used to denote mapping. + + // Not in original libyaml. + yaml_BINARY_TAG = "tag:yaml.org,2002:binary" + yaml_MERGE_TAG = "tag:yaml.org,2002:merge" + + yaml_DEFAULT_SCALAR_TAG = yaml_STR_TAG // The default scalar tag is !!str. + yaml_DEFAULT_SEQUENCE_TAG = yaml_SEQ_TAG // The default sequence tag is !!seq. + yaml_DEFAULT_MAPPING_TAG = yaml_MAP_TAG // The default mapping tag is !!map. +) + +type yaml_node_type_t int + +// Node types. +const ( + // An empty node. + yaml_NO_NODE yaml_node_type_t = iota + + yaml_SCALAR_NODE // A scalar node. + yaml_SEQUENCE_NODE // A sequence node. + yaml_MAPPING_NODE // A mapping node. +) + +// An element of a sequence node. +type yaml_node_item_t int + +// An element of a mapping node. +type yaml_node_pair_t struct { + key int // The key of the element. + value int // The value of the element. +} + +// The node structure. +type yaml_node_t struct { + typ yaml_node_type_t // The node type. + tag []byte // The node tag. + + // The node data. + + // The scalar parameters (for yaml_SCALAR_NODE). + scalar struct { + value []byte // The scalar value. + length int // The length of the scalar value. + style yaml_scalar_style_t // The scalar style. + } + + // The sequence parameters (for YAML_SEQUENCE_NODE). + sequence struct { + items_data []yaml_node_item_t // The stack of sequence items. + style yaml_sequence_style_t // The sequence style. + } + + // The mapping parameters (for yaml_MAPPING_NODE). + mapping struct { + pairs_data []yaml_node_pair_t // The stack of mapping pairs (key, value). + pairs_start *yaml_node_pair_t // The beginning of the stack. + pairs_end *yaml_node_pair_t // The end of the stack. + pairs_top *yaml_node_pair_t // The top of the stack. + style yaml_mapping_style_t // The mapping style. + } + + start_mark yaml_mark_t // The beginning of the node. + end_mark yaml_mark_t // The end of the node. + +} + +// The document structure. +type yaml_document_t struct { + + // The document nodes. + nodes []yaml_node_t + + // The version directive. + version_directive *yaml_version_directive_t + + // The list of tag directives. + tag_directives_data []yaml_tag_directive_t + tag_directives_start int // The beginning of the tag directives list. + tag_directives_end int // The end of the tag directives list. + + start_implicit int // Is the document start indicator implicit? + end_implicit int // Is the document end indicator implicit? + + // The start/end of the document. + start_mark, end_mark yaml_mark_t +} + +// The prototype of a read handler. +// +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. +// +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. +// +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. +type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) + +// This structure holds information about a potential simple key. +type yaml_simple_key_t struct { + possible bool // Is a simple key possible? + required bool // Is a simple key required? + token_number int // The number of the token. + mark yaml_mark_t // The position mark. +} + +// The states of the parser. +type yaml_parser_state_t int + +const ( + yaml_PARSE_STREAM_START_STATE yaml_parser_state_t = iota + + yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE // Expect the beginning of an implicit document. + yaml_PARSE_DOCUMENT_START_STATE // Expect DOCUMENT-START. + yaml_PARSE_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_PARSE_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_PARSE_BLOCK_NODE_STATE // Expect a block node. + yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE // Expect a block node or indentless sequence. + yaml_PARSE_FLOW_NODE_STATE // Expect a flow node. + yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a block sequence. + yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE // Expect an entry of a block sequence. + yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE // Expect an entry of an indentless sequence. + yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_PARSE_BLOCK_MAPPING_KEY_STATE // Expect a block mapping key. + yaml_PARSE_BLOCK_MAPPING_VALUE_STATE // Expect a block mapping value. + yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE // Expect an entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE // Expect a key of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE // Expect a value of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE // Expect the and of an ordered mapping entry. + yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE // Expect an empty value of a flow mapping. + yaml_PARSE_END_STATE // Expect nothing. +) + +func (ps yaml_parser_state_t) String() string { + switch ps { + case yaml_PARSE_STREAM_START_STATE: + return "yaml_PARSE_STREAM_START_STATE" + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return "yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_START_STATE: + return "yaml_PARSE_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return "yaml_PARSE_DOCUMENT_CONTENT_STATE" + case yaml_PARSE_DOCUMENT_END_STATE: + return "yaml_PARSE_DOCUMENT_END_STATE" + case yaml_PARSE_BLOCK_NODE_STATE: + return "yaml_PARSE_BLOCK_NODE_STATE" + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return "yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE" + case yaml_PARSE_FLOW_NODE_STATE: + return "yaml_PARSE_FLOW_NODE_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return "yaml_PARSE_BLOCK_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE" + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE" + case yaml_PARSE_END_STATE: + return "yaml_PARSE_END_STATE" + } + return "" +} + +// This structure holds aliases data. +type yaml_alias_data_t struct { + anchor []byte // The anchor. + index int // The node id. + mark yaml_mark_t // The anchor mark. +} + +// The parser structure. +// +// All members are internal. Manage the structure using the +// yaml_parser_ family of functions. +type yaml_parser_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + + problem string // Error description. + + // The byte about which the problem occurred. + problem_offset int + problem_value int + problem_mark yaml_mark_t + + // The error context. + context string + context_mark yaml_mark_t + + // Reader stuff + + read_handler yaml_read_handler_t // Read handler. + + input_reader io.Reader // File input data. + input []byte // String input data. + input_pos int + + eof bool // EOF flag + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + unread int // The number of unread characters in the buffer. + + newlines int // The number of line breaks since last non-break/non-blank character + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The input encoding. + + offset int // The offset of the current position (in bytes). + mark yaml_mark_t // The mark of the current position. + + // Comments + + head_comment []byte // The current head comments + line_comment []byte // The current line comments + foot_comment []byte // The current foot comments + tail_comment []byte // Foot comment that happens at the end of a block. + stem_comment []byte // Comment in item preceding a nested structure (list inside list item, etc) + + comments []yaml_comment_t // The folded comments for all parsed tokens + comments_head int + + // Scanner stuff + + stream_start_produced bool // Have we started to scan the input stream? + stream_end_produced bool // Have we reached the end of the input stream? + + flow_level int // The number of unclosed '[' and '{' indicators. + + tokens []yaml_token_t // The tokens queue. + tokens_head int // The head of the tokens queue. + tokens_parsed int // The number of tokens fetched from the queue. + token_available bool // Does the tokens queue contain a token ready for dequeueing. + + indent int // The current indentation level. + indents []int // The indentation levels stack. + + simple_key_allowed bool // May a simple key occur at the current position? + simple_keys []yaml_simple_key_t // The stack of simple keys. + simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number + + // Parser stuff + + state yaml_parser_state_t // The current parser state. + states []yaml_parser_state_t // The parser states stack. + marks []yaml_mark_t // The stack of marks. + tag_directives []yaml_tag_directive_t // The list of TAG directives. + + // Dumper stuff + + aliases []yaml_alias_data_t // The alias data. + + document *yaml_document_t // The currently parsed document. +} + +type yaml_comment_t struct { + + scan_mark yaml_mark_t // Position where scanning for comments started + token_mark yaml_mark_t // Position after which tokens will be associated with this comment + start_mark yaml_mark_t // Position of '#' comment mark + end_mark yaml_mark_t // Position where comment terminated + + head []byte + line []byte + foot []byte +} + +// Emitter Definitions + +// The prototype of a write handler. +// +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. +// +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. +// +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. +// +type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error + +type yaml_emitter_state_t int + +// The emitter states. +const ( + // Expect STREAM-START. + yaml_EMIT_STREAM_START_STATE yaml_emitter_state_t = iota + + yaml_EMIT_FIRST_DOCUMENT_START_STATE // Expect the first DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_START_STATE // Expect DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_EMIT_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a flow sequence. + yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE // Expect the next item of a flow sequence, with the comma already written out + yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE // Expect an item of a flow sequence. + yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE // Expect the next key of a flow mapping, with the comma already written out + yaml_EMIT_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a block sequence. + yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE // Expect an item of a block sequence. + yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_KEY_STATE // Expect the key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_VALUE_STATE // Expect a value of a block mapping. + yaml_EMIT_END_STATE // Expect nothing. +) + +// The emitter structure. +// +// All members are internal. Manage the structure using the @c yaml_emitter_ +// family of functions. +type yaml_emitter_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + problem string // Error description. + + // Writer stuff + + write_handler yaml_write_handler_t // Write handler. + + output_buffer *[]byte // String output data. + output_writer io.Writer // File output data. + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The stream encoding. + + // Emitter stuff + + canonical bool // If the output is in the canonical style? + best_indent int // The number of indentation spaces. + best_width int // The preferred width of the output lines. + unicode bool // Allow unescaped non-ASCII characters? + line_break yaml_break_t // The preferred line break. + + state yaml_emitter_state_t // The current emitter state. + states []yaml_emitter_state_t // The stack of states. + + events []yaml_event_t // The event queue. + events_head int // The head of the event queue. + + indents []int // The stack of indentation levels. + + tag_directives []yaml_tag_directive_t // The list of tag directives. + + indent int // The current indentation level. + + flow_level int // The current flow level. + + root_context bool // Is it the document root context? + sequence_context bool // Is it a sequence context? + mapping_context bool // Is it a mapping context? + simple_key_context bool // Is it a simple mapping key context? + + line int // The current line. + column int // The current column. + whitespace bool // If the last character was a whitespace? + indention bool // If the last character was an indentation character (' ', '-', '?', ':')? + open_ended bool // If an explicit document end is required? + + space_above bool // Is there's an empty line above? + foot_indent int // The indent used to write the foot comment above, or -1 if none. + + // Anchor analysis. + anchor_data struct { + anchor []byte // The anchor value. + alias bool // Is it an alias? + } + + // Tag analysis. + tag_data struct { + handle []byte // The tag handle. + suffix []byte // The tag suffix. + } + + // Scalar analysis. + scalar_data struct { + value []byte // The scalar value. + multiline bool // Does the scalar contain line breaks? + flow_plain_allowed bool // Can the scalar be expessed in the flow plain style? + block_plain_allowed bool // Can the scalar be expressed in the block plain style? + single_quoted_allowed bool // Can the scalar be expressed in the single quoted style? + block_allowed bool // Can the scalar be expressed in the literal or folded styles? + style yaml_scalar_style_t // The output style. + } + + // Comments + head_comment []byte + line_comment []byte + foot_comment []byte + tail_comment []byte + + key_line_comment []byte + + // Dumper stuff + + opened bool // If the stream was already opened? + closed bool // If the stream was already closed? + + // The information associated with the document nodes. + anchors *struct { + references int // The number of references. + anchor int // The anchor id. + serialized bool // If the node has been emitted? + } + + last_anchor_id int // The last assigned anchor id. + + document *yaml_document_t // The currently emitted document. +} diff --git a/scraper-go/vendor/gopkg.in/yaml.v3/yamlprivateh.go b/scraper-go/vendor/gopkg.in/yaml.v3/yamlprivateh.go new file mode 100644 index 00000000..e88f9c54 --- /dev/null +++ b/scraper-go/vendor/gopkg.in/yaml.v3/yamlprivateh.go @@ -0,0 +1,198 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +const ( + // The size of the input raw buffer. + input_raw_buffer_size = 512 + + // The size of the input buffer. + // It should be possible to decode the whole raw buffer. + input_buffer_size = input_raw_buffer_size * 3 + + // The size of the output buffer. + output_buffer_size = 128 + + // The size of the output raw buffer. + // It should be possible to encode the whole output buffer. + output_raw_buffer_size = (output_buffer_size*2 + 2) + + // The size of other stacks and queues. + initial_stack_size = 16 + initial_queue_size = 16 + initial_string_size = 16 +) + +// Check if the character at the specified position is an alphabetical +// character, a digit, '_', or '-'. +func is_alpha(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'Z' || b[i] >= 'a' && b[i] <= 'z' || b[i] == '_' || b[i] == '-' +} + +// Check if the character at the specified position is a digit. +func is_digit(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' +} + +// Get the value of a digit. +func as_digit(b []byte, i int) int { + return int(b[i]) - '0' +} + +// Check if the character at the specified position is a hex-digit. +func is_hex(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'F' || b[i] >= 'a' && b[i] <= 'f' +} + +// Get the value of a hex-digit. +func as_hex(b []byte, i int) int { + bi := b[i] + if bi >= 'A' && bi <= 'F' { + return int(bi) - 'A' + 10 + } + if bi >= 'a' && bi <= 'f' { + return int(bi) - 'a' + 10 + } + return int(bi) - '0' +} + +// Check if the character is ASCII. +func is_ascii(b []byte, i int) bool { + return b[i] <= 0x7F +} + +// Check if the character at the start of the buffer can be printed unescaped. +func is_printable(b []byte, i int) bool { + return ((b[i] == 0x0A) || // . == #x0A + (b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E + (b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF + (b[i] > 0xC2 && b[i] < 0xED) || + (b[i] == 0xED && b[i+1] < 0xA0) || + (b[i] == 0xEE) || + (b[i] == 0xEF && // #xE000 <= . <= #xFFFD + !(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF + !(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF)))) +} + +// Check if the character at the specified position is NUL. +func is_z(b []byte, i int) bool { + return b[i] == 0x00 +} + +// Check if the beginning of the buffer is a BOM. +func is_bom(b []byte, i int) bool { + return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF +} + +// Check if the character at the specified position is space. +func is_space(b []byte, i int) bool { + return b[i] == ' ' +} + +// Check if the character at the specified position is tab. +func is_tab(b []byte, i int) bool { + return b[i] == '\t' +} + +// Check if the character at the specified position is blank (space or tab). +func is_blank(b []byte, i int) bool { + //return is_space(b, i) || is_tab(b, i) + return b[i] == ' ' || b[i] == '\t' +} + +// Check if the character at the specified position is a line break. +func is_break(b []byte, i int) bool { + return (b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029) +} + +func is_crlf(b []byte, i int) bool { + return b[i] == '\r' && b[i+1] == '\n' +} + +// Check if the character is a line break or NUL. +func is_breakz(b []byte, i int) bool { + //return is_break(b, i) || is_z(b, i) + return ( + // is_break: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + // is_z: + b[i] == 0) +} + +// Check if the character is a line break, space, or NUL. +func is_spacez(b []byte, i int) bool { + //return is_space(b, i) || is_breakz(b, i) + return ( + // is_space: + b[i] == ' ' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Check if the character is a line break, space, tab, or NUL. +func is_blankz(b []byte, i int) bool { + //return is_blank(b, i) || is_breakz(b, i) + return ( + // is_blank: + b[i] == ' ' || b[i] == '\t' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Determine the width of the character. +func width(b byte) int { + // Don't replace these by a switch without first + // confirming that it is being inlined. + if b&0x80 == 0x00 { + return 1 + } + if b&0xE0 == 0xC0 { + return 2 + } + if b&0xF0 == 0xE0 { + return 3 + } + if b&0xF8 == 0xF0 { + return 4 + } + return 0 + +} diff --git a/scraper-go/vendor/modules.txt b/scraper-go/vendor/modules.txt index 053740d7..bc60350e 100644 --- a/scraper-go/vendor/modules.txt +++ b/scraper-go/vendor/modules.txt @@ -1,15 +1,32 @@ # github.com/PuerkitoBio/goquery v1.12.0 ## explicit; go 1.25.0 github.com/PuerkitoBio/goquery +# github.com/alicebob/miniredis/v2 v2.38.0 +## explicit; go 1.17 +github.com/alicebob/miniredis/v2 +github.com/alicebob/miniredis/v2/fpconv +github.com/alicebob/miniredis/v2/geohash +github.com/alicebob/miniredis/v2/gopher-json +github.com/alicebob/miniredis/v2/hyperloglog +github.com/alicebob/miniredis/v2/metro +github.com/alicebob/miniredis/v2/proto +github.com/alicebob/miniredis/v2/server +github.com/alicebob/miniredis/v2/size # github.com/andybalholm/cascadia v1.3.3 ## explicit; go 1.16 github.com/andybalholm/cascadia # github.com/cespare/xxhash/v2 v2.3.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 +# github.com/davecgh/go-spew v1.1.1 +## explicit +github.com/davecgh/go-spew/spew # github.com/joho/godotenv v1.5.1 ## explicit; go 1.12 github.com/joho/godotenv +# github.com/pmezard/go-difflib v1.0.0 +## explicit +github.com/pmezard/go-difflib/difflib # github.com/redis/go-redis/v9 v9.19.0 ## explicit; go 1.24 github.com/redis/go-redis/v9 @@ -28,6 +45,17 @@ github.com/redis/go-redis/v9/internal/routing github.com/redis/go-redis/v9/internal/util github.com/redis/go-redis/v9/maintnotifications github.com/redis/go-redis/v9/push +# github.com/stretchr/testify v1.11.1 +## explicit; go 1.17 +github.com/stretchr/testify/assert +github.com/stretchr/testify/assert/yaml +github.com/stretchr/testify/require +# github.com/yuin/gopher-lua v1.1.1 +## explicit; go 1.17 +github.com/yuin/gopher-lua +github.com/yuin/gopher-lua/ast +github.com/yuin/gopher-lua/parse +github.com/yuin/gopher-lua/pm # go.uber.org/atomic v1.11.0 ## explicit; go 1.18 go.uber.org/atomic @@ -42,3 +70,6 @@ golang.org/x/sync/singleflight ## explicit; go 1.25.0 golang.org/x/text/transform golang.org/x/text/unicode/norm +# gopkg.in/yaml.v3 v3.0.1 +## explicit +gopkg.in/yaml.v3 From 130ffea110c11e3db3bbe12dfdd8f40d14961421 Mon Sep 17 00:00:00 2001 From: hltav Date: Fri, 5 Jun 2026 10:28:21 -0300 Subject: [PATCH 2/2] =?UTF-8?q?chore(pipeline):=20remove=20implementa?= =?UTF-8?q?=C3=A7=C3=A3o=20antiga=20de=20IndexJobsInValkey=20comentada?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scraper-go/internal/pipeline/pipeline.go | 79 ------------------------ 1 file changed, 79 deletions(-) diff --git a/scraper-go/internal/pipeline/pipeline.go b/scraper-go/internal/pipeline/pipeline.go index b684c177..a5290dfd 100644 --- a/scraper-go/internal/pipeline/pipeline.go +++ b/scraper-go/internal/pipeline/pipeline.go @@ -87,85 +87,6 @@ func Run(ctx context.Context, adapterList []adapters.Adapter, req models.ScrapeR return dedup.DedupeJobs(allJobs) } -// func IndexJobsInValkey(ctx context.Context, rdb *redis.Client, jobs []models.Job, keywords []string) { -// if rdb == nil || len(jobs) == 0 { -// return -// } - -// globalIndexKey := "scraper:jobs:index" -// indexTTL := 72 * time.Hour - -// // Limpa os índices de keyword anteriores (composto + sub-termos) -// for _, kw := range keywords { -// sanitizedKw := strings.ToLower(strings.TrimSpace(kw)) -// if sanitizedKw == "" { -// continue -// } - -// normalizedForDel := strings.ReplaceAll(sanitizedKw, "/", " ") -// normalizedForDel = strings.Join(strings.Fields(normalizedForDel), " ") -// rdb.Del(ctx, fmt.Sprintf("scraper:jobs:keyword:%s", normalizedForDel)) - -// for _, term := range strings.Fields(normalizedForDel) { -// rdb.Del(ctx, fmt.Sprintf("scraper:jobs:keyword:%s", term)) -// } -// } - -// for _, job := range jobs { -// // ✅ Usa o ID estável como membro de todos os Sets, -// // igual ao que jobstore.SaveBatch faz — nunca a URL bruta -// id := jobstore.StableID(&job) -// if id == "" { -// continue -// } - -// // Índice global: apenas IDs (sem duplicar com URLs) -// rdb.SAdd(ctx, globalIndexKey, id) - -// titleLower := strings.ToLower(job.Title) -// descLower := strings.ToLower(job.Description) - -// for _, kw := range keywords { -// sanitizedKw := strings.ToLower(strings.TrimSpace(kw)) -// if sanitizedKw == "" { -// continue -// } - -// normalizedKw := strings.ReplaceAll(sanitizedKw, "/", " ") -// subTerms := strings.Fields(normalizedKw) - -// // Keyword composta: indexa só se todos os sub-termos aparecem na vaga -// matchAll := true -// for _, term := range subTerms { -// if !strings.Contains(titleLower, term) && !strings.Contains(descLower, term) { -// matchAll = false -// break -// } -// } - -// if matchAll { -// fullKey := fmt.Sprintf("scraper:jobs:keyword:%s", strings.Join(subTerms, " ")) -// rdb.SAdd(ctx, fullKey, id) -// rdb.Expire(ctx, fullKey, indexTTL) -// } - -// // Sub-termos individuais: indexa cada um independentemente -// for _, term := range subTerms { -// if term == "" { -// continue -// } -// if strings.Contains(titleLower, term) || strings.Contains(descLower, term) { -// termKey := fmt.Sprintf("scraper:jobs:keyword:%s", term) -// rdb.SAdd(ctx, termKey, id) -// rdb.Expire(ctx, termKey, indexTTL) -// } -// } -// } -// } - -// rdb.Expire(ctx, globalIndexKey, indexTTL) -// slog.Info("Valkey invertido atualizado", "total_vagas", len(jobs)) -// } func IndexJobsInValkey(ctx context.Context, rdb *redis.Client, jobs []models.Job, keywords []string) { if rdb == nil || len(jobs) == 0 { return