From 610f45b322780d2480e0ca67bb17db410d78f607 Mon Sep 17 00:00:00 2001 From: Adnan Hussain <61515575+adnanrhussain@users.noreply.github.com> Date: Mon, 2 Mar 2026 10:39:35 -0800 Subject: [PATCH 1/9] chore: Extract prompts from python to text files for shared infra (#10) * chore: Extract prompts from python to text files for shared infra --- evals/prompts/CHANGELOG.md | 40 +++++ .../grade-level-appropriateness/system.txt | 10 ++ .../grade-level-appropriateness/user.txt | 120 +++++++++++++++ .../sentence-structure/analysis-system.txt | 1 + .../sentence-structure/analysis-user.txt | 38 +++++ .../sentence-structure/complexity-system.txt | 1 + .../sentence-structure/complexity-user.txt | 24 +++ .../sentence-structure/rubric-grade-3.txt | 35 +++++ .../sentence-structure/rubric-grade-4.txt | 36 +++++ .../sentence-structure/rubric-grades-5-12.txt | 5 + .../vocabulary/background-knowledge.txt | 126 ++++++++++++++++ .../prompts/vocabulary/grades-3-4-system.txt | 54 +++++++ evals/prompts/vocabulary/grades-3-4-user.txt | 14 ++ .../vocabulary/other-grades-system.txt | 9 ++ .../prompts/vocabulary/other-grades-user.txt | 139 ++++++++++++++++++ 15 files changed, 652 insertions(+) create mode 100644 evals/prompts/CHANGELOG.md create mode 100644 evals/prompts/grade-level-appropriateness/system.txt create mode 100644 evals/prompts/grade-level-appropriateness/user.txt create mode 100644 evals/prompts/sentence-structure/analysis-system.txt create mode 100644 evals/prompts/sentence-structure/analysis-user.txt create mode 100644 evals/prompts/sentence-structure/complexity-system.txt create mode 100644 evals/prompts/sentence-structure/complexity-user.txt create mode 100644 evals/prompts/sentence-structure/rubric-grade-3.txt create mode 100644 evals/prompts/sentence-structure/rubric-grade-4.txt create mode 100644 evals/prompts/sentence-structure/rubric-grades-5-12.txt create mode 100644 evals/prompts/vocabulary/background-knowledge.txt create mode 100644 evals/prompts/vocabulary/grades-3-4-system.txt create mode 100644 evals/prompts/vocabulary/grades-3-4-user.txt create mode 100644 evals/prompts/vocabulary/other-grades-system.txt create mode 100644 evals/prompts/vocabulary/other-grades-user.txt diff --git a/evals/prompts/CHANGELOG.md b/evals/prompts/CHANGELOG.md new file mode 100644 index 0000000..5298060 --- /dev/null +++ b/evals/prompts/CHANGELOG.md @@ -0,0 +1,40 @@ +# Prompts Changelog + +All notable changes to the evaluator prompt files will be documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + +## [1.2.0] - 2026-02-19 + +### Added +- `vocabulary/other-grades-system.txt` — system prompt for Vocabulary evaluator (grades 5–12) +- `vocabulary/other-grades-user.txt` — user prompt for Vocabulary evaluator (grades 5–12) + +### Changed +- `vocabulary/grades-3-4-system.txt` — updated to reference "Qualitative Text Complexity rubric (SAP)" + +## [1.1.0] - 2026-02-18 + +### Added +- `sentence-structure/rubric-grades-5-12.txt` — SS complexity scoring rubric for grades 5–12 + +### Changed +- `sentence-structure/complexity-system.txt` — updated to reference "Qualitative Text Complexity rubric (SAP)" +- `sentence-structure/analysis-user.txt` — added `Basic Complex` and `Advanced Complex` to the sentence type definitions + +## [1.0.0] - 2025-09-23 + +### Added +- `grade-level-appropriateness/system.txt` — system prompt for the GLA evaluator +- `grade-level-appropriateness/user.txt` — user prompt for the GLA evaluator +- `sentence-structure/analysis-system.txt` — system prompt for SS sentence analysis +- `sentence-structure/analysis-user.txt` — user prompt for SS sentence analysis +- `sentence-structure/complexity-system.txt` — system prompt for SS complexity scoring +- `sentence-structure/complexity-user.txt` — user prompt for SS complexity scoring +- `sentence-structure/rubric-grade-3.txt` — SS complexity scoring rubric for grade 3 +- `sentence-structure/rubric-grade-4.txt` — SS complexity scoring rubric for grade 4 +- `vocabulary/background-knowledge.txt` — background knowledge context for the Vocabulary evaluator +- `vocabulary/grades-3-4-system.txt` — system prompt for Vocabulary evaluator (grades 3–4) +- `vocabulary/grades-3-4-user.txt` — user prompt for Vocabulary evaluator (grades 3–4) diff --git a/evals/prompts/grade-level-appropriateness/system.txt b/evals/prompts/grade-level-appropriateness/system.txt new file mode 100644 index 0000000..dcb8be2 --- /dev/null +++ b/evals/prompts/grade-level-appropriateness/system.txt @@ -0,0 +1,10 @@ + +You are an expert in English literature education for K-12. +Your job is to help evaluate the grade level appropriateness of a given text. + +You will be given a text and you should determine which grade level the text is appropriate for (grade levels include: K-1, 2-3, 4-5, 6-8, 9-10, 11-CCR) + +IMPORTANT: You should pay attention to the vocabulary used, topics of the text and readability of text. + +Please first reason out loud about the vocabulary complexity of the text and then provide an answer between grade level options: K-1, 2-3, 4-5, 6-8, 9-10, 11-CCR. + diff --git a/evals/prompts/grade-level-appropriateness/user.txt b/evals/prompts/grade-level-appropriateness/user.txt new file mode 100644 index 0000000..da0a751 --- /dev/null +++ b/evals/prompts/grade-level-appropriateness/user.txt @@ -0,0 +1,120 @@ + +Use these steps to determine appropriate grade level for a text: +1. Calculate word count and Flesch-Kincaid Grade Level of the text, and generate a grade band. +Here are the bands guideline for word count + +2-3: 200-800 words +4-5: 200-800 words +6-8: 400-1000 words +9-10: 500-1500 words +11-12: 1501 words and more + +Here is the formula for Flesch-Kincaid Grade Level: +Flesch-Kincaid Grade Level = 0.39 * (total words / total sentences) + 11.8 * (total syllables / total words) - 15.59 + + +2. Determine the qualitative complexity using this text complexity rubric: +TEXT STRUCTURE + +Exceedingly Complex + • Deep, intricate, often ambiguous connections between many ideas/processes/events + • Organization is intricate or discipline-specific + • Text features are essential for understanding + • Graphics are intricate, extensive, and integral to meaning; may convey unique information + +Very Complex + • Expanded ideas/processes/events with implicit or subtle connections + • Organization may have multiple pathways or discipline-specific traits + • Text features directly enhance understanding + • Graphics support or are integral to understanding + +Moderately Complex + • Some implicit/subtle connections between ideas/events + • Organization is evident and generally sequential or chronological + • Text features enhance understanding + • Graphics are mostly supplementary + +Slightly Complex + • Explicit and clear connections between ideas/events + • Organization is chronological, sequential, or predictable + • Text features help navigation but are not essential + • Graphics are simple, not necessary, but may assist understanding + +⸻ + +LANGUAGE FEATURES + +Exceedingly Complex + • Dense, abstract, ironic, and/or figurative language + • Complex, unfamiliar, archaic, subject-specific, or ambiguous vocabulary + • Mainly complex sentences with multiple subordinate clauses and transitions + +Very Complex + • Fairly complex; some abstract, ironic, and/or figurative language + • Some unfamiliar, archaic, or overly academic vocabulary + • Many complex sentences with subordinate phrases/clauses + +Moderately Complex + • Mostly explicit language with some complex meaning + • Mostly familiar and conversational vocabulary + • Primarily simple and compound sentences, with some complex ones + +Slightly Complex + • Explicit, literal, straightforward language + • Contemporary, familiar, conversational vocabulary + • Mainly simple sentences + +⸻ + +PURPOSE + +Exceedingly Complex + • Subtle, intricate, and difficult to determine + • Includes many theoretical or abstract elements + +Very Complex + • Implicit or subtle, fairly easy to infer + • More theoretical or abstract than concrete + +Moderately Complex + • Implied but easy to identify based on context or source + +Slightly Complex + • Explicitly stated, clear, concrete, and narrowly focused + +⸻ + +KNOWLEDGE DEMANDS + +Exceedingly Complex + • Requires extensive discipline-specific or theoretical knowledge + • Many references/allusions to other texts or ideas + +Very Complex + • Requires moderate discipline-specific knowledge + • Some references/allusions to other texts or ideas + +Moderately Complex + • Requires common knowledge and some discipline-specific knowledge + • Few references/allusions + +Slightly Complex + • Requires everyday, practical knowledge + • No references/allusions + +3. Background knowledge: +At which grade level would student have enough background knowledge to understand the text? + +4. Use your judgement of the above three steps. First use the quantitative signal to get first signal of the appropriate grade level range, then use qualitative analysis to refine your decisions and consider if student at such grade will have enough background knowledge to arrive at a final grade level band. Also consider if the text can be for a lower grade with additional scaffolding. + + +{text} + + +When providing your response, first think out loud of your reasoning and then provide your answer from one of the grade band options above. Your reasoning and answer needs to be in JSON format. Strictly follow the following format for your response. + +Your final answer should be in the "grade" property for the target grade band for the text aimed for independent reading. If there is alternative appropriate grade students can read and comprehend with scaffold (eg. picture, graph, additional context, etc) or for read-aloud purposes for lower grade, provide it in the "alternative_grade" property and provide the types of scaffolding in the "scaffolding_needed" property. + +In your reasoning, provide numbered bullet points for each of the analyses in each of the 3 steps. At the end, give me the 4th bullet point called "synthesis" to summarize your analysis from the above 3 steps that help you arrive at the final decision. + +{format_instructions} diff --git a/evals/prompts/sentence-structure/analysis-system.txt b/evals/prompts/sentence-structure/analysis-system.txt new file mode 100644 index 0000000..67acbd0 --- /dev/null +++ b/evals/prompts/sentence-structure/analysis-system.txt @@ -0,0 +1 @@ +You are an expert in grammar and literacy. \ No newline at end of file diff --git a/evals/prompts/sentence-structure/analysis-user.txt b/evals/prompts/sentence-structure/analysis-user.txt new file mode 100644 index 0000000..230a95d --- /dev/null +++ b/evals/prompts/sentence-structure/analysis-user.txt @@ -0,0 +1,38 @@ + +# Task +I am going to give you a text, and I need you to look through the text sentence-by-sentence to perform a comprehensive grammatical analysis. Use the computational counts as a reference; they can be incorrect in ambiguous cases. + +# Definitions +* Sentences: Count a complete grammatical unit ending in a terminal punctuation mark. +* Words: Count any sequence of characters separated by a space as one word. Treat hyphenated words (e.g., "state-of-the-art") and numbers (e.g., "2025") as single words. +* Independent Clauses: Clauses that can stand alone as a complete sentence. +* Subordinate Clauses: Clauses that are dependent on the main clause and cannot stand alone as a complete sentence. +* Simple Sentences: Sentences with one independent clause and no subordinate clauses. +* Compound Sentences: Sentences with two or more independent clauses and no subordinate clauses. +* Complex Sentences: Sentences with one independent clause and at least one subordinate clause. +* Compound-Complex Sentences: Sentences with two or more independent clauses and at least one subordinate clause. +* Other / Non-Canonical Sentences: Sentences that cannot be reliably classified as simple, compound, complex, or compound-complex (e.g., sentence fragments, run-ons, elliptical responses, headlines, imperatives lacking an explicit subject, or stylized dialogue tags). +* Subordinate Clauses: Clauses that are dependent on the main clause and cannot stand alone as a complete sentence. +* Embedded Clauses: Clauses that are nested within another clause. +* Prepositional Phrases: Phrases that begin with a preposition and end with a noun phrase. +* Participle Phrases: Phrases that begin with a participle and end with a noun phrase. +* Appositive Phrases: Phrases that rename or identify a noun phrase. +* Simple Transitions: Basic coordinating conjunctions and chronological adverbs. Examples: 'and', 'but', 'or', 'so', 'then', 'next', 'first'. +* Sophisticated Transitions: Conjunctive adverbs and phrases signaling logical relationships. Examples: 'however', 'therefore', 'consequently', 'as a result', 'for example', 'although'. +* One-Concept Sentence: A sentence with ZERO subordinate clauses AND ZERO transition words/phrases (neither simple nor sophisticated). +* Multi-Concept Sentence: Any sentence that has ≥1 subordinate clause OR ≥1 transition word/phrase (or both). +* Basic Complex Sentences: Sentences with exactly one independent clause and at one dependent (subordinate) clause. +* Advanced Complex Sentences: Sentences with two or more of any of those following (can include a mix, doesn't have to be two of the same type) subordinate phrases, clauses, transition words, or any other meaningful "interruptions" to the flow of the sentence (like not-only-but-also constructions, dashes, semicolons, and lengthy appositives). A sentence can be advanced complex if it has just one subordinate phrase or clause alongside a transition phrase, like: "For example, the British favored trade with Hong Kong, assuming favorable trade conditions. + +# Computational Counts +Use these as reference, your internal heuristics can be more reliable. +{ground_truth_counts} + +# Text to Analyze +[BEGIN TEXT] +{text} +[END TEXT] + +IMPORTANT: Your response should be a single JSON object with the following structure. Do not produce anything outside of the JSON object. + +{format_instructions} diff --git a/evals/prompts/sentence-structure/complexity-system.txt b/evals/prompts/sentence-structure/complexity-system.txt new file mode 100644 index 0000000..4303ff8 --- /dev/null +++ b/evals/prompts/sentence-structure/complexity-system.txt @@ -0,0 +1 @@ +You are an expert in grammar and literacy, and understand K-12 and Qualitative Text Complexity rubric (SAP). \ No newline at end of file diff --git a/evals/prompts/sentence-structure/complexity-user.txt b/evals/prompts/sentence-structure/complexity-user.txt new file mode 100644 index 0000000..f380358 --- /dev/null +++ b/evals/prompts/sentence-structure/complexity-user.txt @@ -0,0 +1,24 @@ + +Your task is to perform a text complexity analysis for a Grade {grade} student. You will be given a text excerpt and a set of quantitative sentence-level statistics for that text. + +You must integrate both the qualitative aspects of the text and the quantitative statistics to make your final judgment. Do not rely on the numbers alone. + +1. Read the TEXT EXCERPT to understand its topic, conceptual load, and overall structure. +2. Review the TEXT STATISTICS as a guide for complexity level. +3. Synthesize your findings in your reasoning. Explain how the structure (qualitative) interact with the text statistics (quantitative) to determine the complexity. For example, a text with simple sentences might still be complex if the topic is very dense or abstract. + +Your final answer must be one of ["Slightly Complex," "Moderately Complex," "Very Complex", "Exceedingly Complex"]. + +# GRADE {grade} RUBRIC +{rubric} + +# TEXT EXCERPT +[BEGIN TEXT] +{excerpt} +[END TEXT] + +# TEXT STATISTICS +{sentence_features} + +# OUTPUT FORMAT +{format_instructions} diff --git a/evals/prompts/sentence-structure/rubric-grade-3.txt b/evals/prompts/sentence-structure/rubric-grade-3.txt new file mode 100644 index 0000000..a803ab0 --- /dev/null +++ b/evals/prompts/sentence-structure/rubric-grade-3.txt @@ -0,0 +1,35 @@ + + **Instructions for Analysis:** First, evaluate if the text meets the criteria for "Slightly Complex" or "Exceedingly Complex". If it does not fit into these categories, then decide between "Moderately Complex" and "Very Complex". + + **Slightly Complex:** + * **Description:** The text consists of simple, straightforward language and sentence structures. + * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: + * **Sentence Type:** Primarily simple sentences. (`percent_simple_sentences` is typically > 60%). + * **Sentence Length:** Short sentences. (`avg_sentence_length` is typically < 12 words). + * **Subordination:** Very low use of clauses. (`percent_sentences_with_subordinate` is typically < 25%). + + **Moderately Complex:** + * **Description:** The text shows a mix of simple and more complex sentences, introducing some variety in structure without being overly demanding. + * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: + * **Sentence Type:** A balanced mix of sentence types. (`percent_simple_sentences` is typically between 40% and 60%). + * **Sentence Length:** Medium length sentences. (`avg_sentence_length` is typically between 12 and 16 words). + * **Subordination:** A moderate use of clauses. (`percent_sentences_with_subordinate` is typically between 25% and 45%). + + **Very Complex:** + * **Description:** The text features more elaborate sentences with multiple clauses and ideas, requiring more effort from the reader to parse. This is often the default category for grade-level text that isn't simple or exceptionally difficult. + * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: + * **Sentence Type:** Complex structures are common. (`percent_simple_sentences` is a minority, typically < 40%). + * **Sentence Length:** Longer sentences are frequent. (`avg_sentence_length` is typically between 16 and 19 words). + * **Subordination:** Subordinate clauses are a key feature. (`percent_sentences_with_subordinate` is typically > 45%). + + **Exceedingly Complex:** + * **Description:** The text is dense with very long, intricate sentences and a high degree of subordination, making it exceptionally challenging for this grade level. + * **Statistical Guidelines:** The text is "Exceedingly Complex" if it shows an extreme combination of sentence length and structural density. It should meet at least **TWO** of the following criteria, including at least **ONE** from the "Structural Density" group. + * **Structural Density Indicators:** + * High Subordination: `percent_sentences_with_subordinate` is extensive (typically > 50%). + * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is consistently present (typically > 12%). + * High Syntactic Complexity: `percent_compound_complex_sentences` is significant (typically > 15%). + * **Length Indicators:** + * Extreme Sentence Length: `avg_sentence_length` is very long (typically > 19 words). + * Low Simplicity: `percent_simple_sentences` is very low (typically < 30%). + * Concentrated Length: `percent_very_long_sentences` is notable (typically > 10%). diff --git a/evals/prompts/sentence-structure/rubric-grade-4.txt b/evals/prompts/sentence-structure/rubric-grade-4.txt new file mode 100644 index 0000000..6a81c2b --- /dev/null +++ b/evals/prompts/sentence-structure/rubric-grade-4.txt @@ -0,0 +1,36 @@ + + **Instructions for Analysis:** First, evaluate if the text meets the criteria for "Slightly Complex" or "Exceedingly Complex". If it does not fit into these categories, then decide between "Moderately Complex" and "Very Complex". + + **Slightly Complex:** + * **Description:** The text uses clear, direct language with basic sentence structures appropriate for developing readers. + * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: + * **Sentence Type:** Dominated by simple sentences. (`percent_simple_sentences` is typically > 55%). + * **Sentence Length:** Short to medium sentences. (`avg_sentence_length` is typically < 13 words). + * **Subordination:** Infrequent use of clauses. (`percent_sentences_with_subordinate` is typically < 30%). + + **Moderately Complex:** + * **Description:** The text contains a variety of sentence structures, including compound and complex sentences, but remains accessible. + * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: + * **Sentence Type:** A healthy mix of sentence types. (`percent_simple_sentences` is typically between 40% and 55%). + * **Sentence Length:** Medium length sentences. (`avg_sentence_length` is typically between 13 and 17 words). + * **Subordination:** A moderate number of clauses. (`percent_sentences_with_subordinate` is typically between 30% and 50%). + + **Very Complex:** + * **Description:** The text is characterized by longer sentences and the regular use of dependent clauses, requiring readers to track multiple ideas. This is the default for challenging, on-grade-level texts. + * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: + * **Sentence Type:** Simple sentences are a clear minority. (`percent_simple_sentences` is typically < 40%). + * **Sentence Length:** Sentences are consistently long. (`avg_sentence_length` is typically between 17 and 22 words). + * **Subordination:** Subordination is a major feature. (`percent_sentences_with_subordinate` is typically > 50%). + * **Multiple Subordination:** Sentences with multiple clauses appear more often. (`percent_sentences_with_multiple_subordinates` is typically > 8%). + + **Exceedingly Complex:** + * **Description:** The text's structure is highly sophisticated and dense, marked by extensive use of embedded clauses and long, flowing sentences that are well above grade-level expectations. + * **Statistical Guidelines:** A text is "Exceedingly Complex" if its structure is highly sophisticated and dense. It should meet at least **TWO** of the following criteria, including at least **ONE** from the "Structural Density" group. + * **Structural Density Indicators:** + * High Subordination: `percent_sentences_with_subordinate` is very high (typically > 60%). + * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is high and consistent (typically > 15%). + * High Syntactic Complexity: `percent_compound_complex_sentences` is a notable feature (typically > 20%). + * **Length Indicators:** + * Extreme Sentence Length: `avg_sentence_length` is exceptionally long (typically > 22 words). + * Low Simplicity: `percent_simple_sentences` is very low (typically < 25%). + * Concentrated Length: `percent_very_long_sentences` is significant (typically > 15%). diff --git a/evals/prompts/sentence-structure/rubric-grades-5-12.txt b/evals/prompts/sentence-structure/rubric-grades-5-12.txt new file mode 100644 index 0000000..b69c845 --- /dev/null +++ b/evals/prompts/sentence-structure/rubric-grades-5-12.txt @@ -0,0 +1,5 @@ + + **Slightly Complex:** A text is in the Slightly Complex bucket if it has at least 50% simple sentences. If it doesn't, the text is a higher level of complexity. If the % of simple sentences is >= 50% and the % of compound sentences is >= 20%, the text is Moderately Complex, otherwise, the text is Slightly Complex. Slightly Complex texts NEVER have advanced complex sentences — the presence of an advanced complex sentence always leads to a higher level of complexity than Slightly. + **For Moderately Complex:** These texts can take on any distribution of sentence types as long as there aren't more than 2 advanced complex sentences and as long as there aren't so many simple sentences that the text becomes Slightly Complex. That means Moderately Complex texts may have many simple sentences (although not so many that the text is Slightly Complex), compound sentences, and/or basic complex sentences. It's also possible for a moderately complex text to contain one or two advanced complex sentences, as long as there aren't more than 2. If there are more than 2, then the text is either Very or Exceedingly complex. + **Very Complex:** These texts contain 3 or more advanced complex sentences (unless the percentage of advanced complex sentences is >= 65)%, in which case the text becomes Exceedingly Complex). They may still contain many simple, compound, and basic complex sentences, but a text is not Very Complex unless there are 3 or more advanced complex sentences. + **Exceedingly Complex:** These texts have 65%+ of their sentences being advanced complex sentences. diff --git a/evals/prompts/vocabulary/background-knowledge.txt b/evals/prompts/vocabulary/background-knowledge.txt new file mode 100644 index 0000000..868cc7f --- /dev/null +++ b/evals/prompts/vocabulary/background-knowledge.txt @@ -0,0 +1,126 @@ + +Review the following text, which is an educational text written for students in the following grade band: {grade}. + +Your job is to give me a background knowledge assumption; that is: what topics, if any, from the text students are likely to be familiar with based on a standard progression of topics in US public school education, as well as topics, if any the student is not likely to be familiar with. + +Make sure your response is concise (between 1 - 3 lines max) and is about the topics themselves, not about any other aspect of the text (e.g. flowery language, complicated sentence structure, etc.). + +Here's an example: +[START EXAMPLE] +Grade Band: 11th +Text: I went to the woods because I wished to live deliberately, to front only the essential facts of life, and see if I could not +learn what it had to teach, and not, when I came to die, discover that I had not lived. I did not wish to live what was +not life, living is so dear; nor did I wish to practise resignation, unless it was quite necessary. I wanted to live deep and suck out all the marrow of life, to live so sturdily and Spartan-like as to put to rout all that was not life, to cut a broad swath and shave close, to drive life into a corner, and reduce it to its lowest terms, and, if it proved to be mean, why then to get the whole and genuine meanness of it, and publish its meanness to the world; or if it were sublime, to +know it by experience, and be able to give a true account of it in my next excursion. For most men, it appears to me, +are in a strange uncertainty about it, whether it is of the devil or of God, and have somewhat hastily concluded that it +is the chief end of man here to "glorify God and enjoy him forever." + +Background Knowledge Assumption: Assume they've studied American Transcendentalists like Thoreau and Emerson, including the mid-19th-century context of nature-focused philosophy. +[END EXAMPLE] + +You should assume that the student is an average US public school who is learning from common core curriculum. When you respond, just respond with the background knowledge assumption and nothing else. + +You can use the following list of topics that we know are covered for each grade level, although use your best judgement if you know there are other topics out there that students are likely to have covered. And this doesn't cover higher grade levels, so you'll have to again use your judgement for, say, what background knowledge a 9th grader is likely to have: +[BEGIN TOPICS] +[ + K: [ + "Toys and Play", "Weather Wonders", "Trees are Alive", "Enjoying and Appreciating Trees", + "The Five Senses: How do our senses help us learn?", "Once Upon a Farm: What makes a good story?", + "America, Then and Now: How has life in America changed over time?", "The Continents: What makes the world fascinating?", + "Needs of Plants and Animals", "Pushes and Pulls", "Sunlight and Weather", "Learning and Working Together", + "How Do People Learn and Work Together?", "Where Do We Live?", "What Does it Mean to Be an American?", + "How Has Our World Changed?", "Why Do People Have Jobs?" + ], + 1: [ + "Tools and Work", "A Study of the Sun, Moon, and Stars", "Birds' Amazing Bodies", "Caring for Birds", + "A World of Books: How do books change lives around the world?", "Creature Features: What can we discover about animals' unique features?", + "Powerful Forces: How do people respond to the powerful force of the wind?", "Cinderella Stories: Why do people around the world admire Cinderella?", + "Animal and Plant Defenses", "Light and Sounds", "Spinning Earth", "Our Place in the World", + "What Are the Rights and Responsibilities of Citizens?", "How Can We Describe Where We Live?", + "How Do We Celebrate Our Country?", "How Does the Past Shape Our Lives?", "Why Do People Work?" + ], + 2: [ + "Schools and Community", "Fossils Tell of Earth's Changes", "The Secret World of Pollination", "Providing for Pollinators", + "A Season of Change: How does change impact people and nature?", "The American West: What was life like in the West for early Americans?", + "Civil Rights Heroes: How can people respond to injustice?", "Good Eating: How does food nourish us?", + "Plant and Animal Relationships", "Properties of Matter", "Changing Landforms", "Exploring Who We Are", + "Why Is It Important to Learn About the Past?", "How Does Geography Help Us Understand Our World?", + "How Do We Get What We Want and Need?", "Why Do We Need Government?", "How Can People Make a Difference in Our World?" + ], + "3": [ + "Overcoming Learning Challenges Near and Far", "Adaptations and the Wide World of Frogs", "Exploring Literary Classics", + "Water Around the World", "Ocean/Sea Exploration", "Outer Space", "Immigration", "Art/Being an Artist", + "Balancing Forces", "Inheritance and Traits", "Environments and Survival", "Weather and Climate", + "Communities", "Why Does It Matter Where We Live?", "What Is Our Relationship With Our Environment?", + "What Makes a Community Unique?", "How Does the Past Impact the Present?", "Why Do Governments and Citizens Need Each Other?", + "How Do People in a Community Meet Their Wants and Needs?" + ], + 4: [ + "Poetry", "Animal Defense Mechanisms", "The American Revolution", + "Responding to Inequality: Ratifying the 19th Amendment (covers gender and racial inequality)", + "A Great Heart: What does it mean to have a great heart, literally and figuratively?", + "Extreme Settings: How does a challenging setting or physical environment change a person?", + "American Revolution/Multiple Perspectives", "Myths/Myth Making", "Energy Conversions", "Vision and Light", + "Earth's Features", "Waves, Energy, and Information", "Regions of the United States", + "How Does America Use Its Strengths and Face Its Challenges?", "Why Have People Moved to and From the Northeast?", + "How Has the Southeast Changed Over Time?", "How Does the Midwest Reflect the Spirit of America?", + "How Does the Southwest Reflect Its Diverse Past and Unique Environment?", "What Draws People to the West?" + ], + 5: [ + "Human Rights", "Biodiversity in the Rainforest", "Athlete Leaders of Social Change", + "Impact of Natural Disasters", "Cultures in Conflict: How do cultural beliefs and values guide people?", + "Word Play: How and why do writers play with words?", "A War Between Us: How did the Civil War impact people?", + "Breaking Barriers: How can sports influence individuals and societies?", "Patterns of Earth and Sky", + "Modeling Matter", "The Earth System", "Ecosystem Restoration", "U.S. History: Making a New Nation", + "How Were the Lives of Native Peoples Influenced by Where They Lived?", + "What Happened When Diverse Cultures Crossed Paths?", "What Is the Impact of People Settling in a New Place?", + "Why Would a Nation Want to Become Independent?", "What Does the Revolutionary Era Tell Us About Our Nation Today?", + "How Does the Constitution Help Us Understand What It Means to Be an American?", + "What Do the Early Years of the United States Reveal About the Character of the Nation?", + "What Was the Effect of the Civil War on U.S. Society?" + ], + 6: [ + "Greek Mythology", "Critical Problems and Design Solutions", "American Indian Boarding Schools", + "Remarkable Accomplishments in Space Science", "Resilience in the Great Depression: How can enduring tremendous hardship contribute to personal transformation?", + "A Hero's Journey: What is the significance and power of the hero's journey?", + "Narrating the Unknown: How did the social and environmental factors in the unknown world of Jamestown shape its development and decline?", + "Courage in Crisis: How can the challenges of a hostile environment inspire heroism?", + "Microbiome", "Metabolism", "Metabolism Engineering", "Traits and Reproduction", "Thermal Energy", + "Ocean, Atmosphere, and Climate", "Weather Patterns", "Earth's Changing Climate", + "Earth's Changing Climate: Engineering Internship", "The First Americans (up to 1492)", + "Exploration and Colonization", "English Colonies", "American Revolution", "First Governments and the Constitution", + "The Early American Republic", "Political and Geographic Changes (1828-1850)", "Life in the North and South (1820-1860)", + "Division and Civil War (1821-1865)", "Reconstruction (1865-1896)", "The West (1858-1896)", + "New Industry and a Changing Society", "Expansion and War", "The 1920s and 1930s", "World War II", + "The Cold War", "Civil Rights and American Society", "America Since the 1970s" + ], + 7: [ + "The Lost Children of Sudan (Genocide, Genocide in Sudan)", "Epidemics", "Harlem Renaissance", "Plastic Pollution", + "Identity in the Middle Ages: How does society both support and limit the development of identity?", + "Americans All: How did World War II affect individuals?", "Language and Power: What is the power of language?", + "Fever: How can times of crisis affect citizens and society?", "Geology on Mars", "Plane Motion", "Plane Motion Engineering", + "Rock Formations", "Phase Change", "Phase Change Engineering", "Chemical Reactions", "Populations and Resources", + "Matter and Energy in Ecosystems", "Early Humans and Agricultural Revolution", "Fertile Crescent", + "Ancient Egypt and Kush", "The Israelites", "Ancient Greece", "Ancient South Asia", "Early China, Korea, and Japan", + "Ancient Rome", "Rise of Christian Kingdoms", "The Americas", "Medieval Europe", "The Rise of Islamic Empires", + "China in the Middle Ages", "Korea and Japan in the Middle Ages", "African Civilizations", "New Ways of Thinking", + "Age of Exploration and Trade", "Revolutions and Empires", "The Modern World" + ], + 8: [ + "Folklore of Latin America", "Food Choices", "The Holocaust", "Japanese American Internment", + "The Poetics and Power of Storytelling: What is the power of storytelling?", + "The Great War: How do literature and art illuminate the effects of World War I?", "What Is Love?", + "Teens as Change Agents: How do people effect social change?", "Harnessing Human Energy", + "Force and Motion", "Force and Motion Engineering", "Magnetic Fields", "Light Waves", "Earth, Moon, and Sun", + "Natural Selection", "Natural Selection Engineering", "Evolutionary History", "The World in Spatial Terms", + "Places and Regions", "Physical Geography", "Population Geography", "Economic Geography", + "Political Geography", "Human-Environment Geography", "What is Economics?", "Markets, Money, and Businesses", + "Government and the Economy", "The Global Economy" + ] +] +[END TOPICS] + +Here is the text: +[BEGIN TEXT] +{text} +[END TEXT] diff --git a/evals/prompts/vocabulary/grades-3-4-system.txt b/evals/prompts/vocabulary/grades-3-4-system.txt new file mode 100644 index 0000000..3fb50a8 --- /dev/null +++ b/evals/prompts/vocabulary/grades-3-4-system.txt @@ -0,0 +1,54 @@ + +You are an expert curriculum designer. Your job is to rate the complexity of a text's vocabulary relative to the grade level. + +You will be given a rubric (with levels from least to most complex: slightly complex, moderately complex, very complex, exceedingly complex) as well as guidelines for interpreting the rubric. +IMPORTANT: You should only pay attention to the vocabulary. Do not evaluate any other element of the text's complexity (e.g. sentence structure, meaning, etc.) + +**Resource 1: Qualitative Text Complexity rubric (SAP)** +1. **Level 1: Slightly complex** + * Original Definition: Vocabulary that is almost entirely not complex: contemporary, conversational, and/or familiar. A very low proportion of complex words (archaic, subject-specific, academic) is OK -- i.e. doesn't need to be 0. + * Summary definition: Overall, vocabulary is easy to understand and does not impede comprehension of the bulk of the text (including main idea and supporting claims). 1-2 quick pauses for processing by the student are ok here! +2. **Level 2: Moderately complex** + * Original Definition: Vocabulary that is mostly not complex: contemporary, conversational, and/or familiar. A low proportion of complex words (archaic, subject-specific, academic) is OK + * Summary definition: Overall, vocabulary generally allows students to comprehend the bulk of the text with little difficulty, though there may be occasional pauses for clarification. Several quick pauses or occasional prolonged pauses may occur. +3. **Level 3: Very complex** + * Original Definition: Vocabulary that is often complex: unfamiliar, archaic, subject-specific, and/or overly academic + * Summary definition: Overall, vocabulary often presents challenges that may slow down comprehension but does not completely block the comprehension of the bulk of the text. +4. **Level 4: Exceedingly complex** + * Original Definition: Vocabulary that is mostly complex: unfamiliar, archaic, subject-specific, and/or overly academic. May be ambiguous or purposefully misleading. + * Summary definition: Overall, vocabulary is so complex that it makes comprehension of the bulk of the text very challenging and requires careful effort to interpret. + +**Resource 2: Flesch-Kincaid Grade Level** +Use the Flesch-Kincaid (FK) Grade Level as light guidance of the approximate grade level based on readability. The metric alone does not provide final information of vocabulary complexity, but a ballpark of the difficulty of the entire text. +* grade 2-3: 1.98-5.34 +* grade 4-5: 4.51-7.73 +* grade 6-8: 6.51-10.34 +* grade 9-10: 8.32-12.12 +* grade 11-College: 10.34-14.2 + +**Guidelines for Interpretation and Reasoning** + +Your reasoning is the most critical part of your analysis. It's not enough to simply count complex words. You must analyze their impact on a student at the specified grade level. Use the following principles to guide your judgment: + +1. **Density and Cumulative Effect:** Do not just count complex words; evaluate their concentration. A short text with a high density of challenging Tier 2 words (e.g., `peculiar`, `mischievous`, `courageous` for a 4th grader) can be more overwhelming than a longer text with a few scattered Tier 3 words. A constant barrage of unfamiliar words can elevate complexity from `very` to `exceedingly`. +2. **Contextual Scaffolding:** Assess how the text supports new vocabulary. + * Are new, complex terms explicitly defined or explained with simple examples (e.g., "volume... to see if it is big enough to hold a liter of food")? + * Is the surrounding language simple and conversational, making the meaning of new words easier to infer? + * Strong scaffolding can lower the complexity rating. A text with many Tier 3 words that are well-explained might only be `moderately complex`. +3. **Abstract vs. Concrete Vocabulary:** Differentiate between words for abstract concepts and words for concrete things. A text built on abstract Tier 2 words (e.g., `relationships`, `performance`, `non-physical`) can be more challenging than a text that introduces Tier 3 labels for concrete things or people (e.g., `Sumerians`, `polonium`). +4. **Conceptual Load:** Consider the cognitive load of the vocabulary. A list of many new, multi-syllabic, conceptually-heavy terms (e.g., `Paleolithic`, `Mesolithic`, `Neolithic` for a 3rd grader) can be `very complex` even if the terms are briefly defined, because the student must process multiple new concepts at once. +5. **Calibrating the Top Levels:** Be precise in your use of `very complex` vs. `exceedingly complex`. + * **Very complex:** The vocabulary creates significant hurdles and slows the reader down, but the main ideas of the text are still accessible with effort. + * **Exceedingly complex:** The vocabulary is so dense, technical, or abstract that it acts as a barrier, making it nearly impossible for the target student to grasp the bulk of the text's meaning without extensive outside help. Reserve this for texts saturated with advanced terminology. +6. **Consider Background Knowledge:** Pay close attention to the provided `student_background_knowledge`. Do not classify a word as complex if the student is likely to be familiar with it (e.g., 'oxygen' for a 3rd grader who has learned about the human body). + +**Final Analysis Format** + +Provide these information as your final analysis: +1. **Complex vocabulary:** + * Tier 2 words: Words that are commonly used in academic settings and more complex than colloquial, or everyday language and often have multiple meanings. + * Tier 3 words: Overly academic or domain-specific words. + * Archaic words: Words, or uses of words that are not commonly used in modern conversational language. E.g., "The jury retired to deliberate on their verdict." The use of "retire" to mean withdrawing to a private place is an archaic use. + * Other complex words: All other words that can increase complexity of the text (e.g., idioms, unfamiliar proper nouns that function as vocabulary). +2. **Vocabulary complexity:** one of: slightly complex, moderately complex, very complex, exceedingly complex +3. **Your reasoning of the complexity:** A detailed explanation of your rating, referencing the principles above. diff --git a/evals/prompts/vocabulary/grades-3-4-user.txt b/evals/prompts/vocabulary/grades-3-4-user.txt new file mode 100644 index 0000000..1759511 --- /dev/null +++ b/evals/prompts/vocabulary/grades-3-4-user.txt @@ -0,0 +1,14 @@ + +Below is the text you need to evaluate. Let's think step by step in order to predict the output of the vocabulary complexity task. + +- It is intended for grade {student_grade_level}. + +- You can assume the student has the following background knowledge about the text — this background knowledge influences which words from the text are familiar versus unfamiliar for the student: {student_background_knowledge} + +- Text Flesch-Kincaid grade level: {fk_level} + +- Text to evaluate: [BEGIN TEXT] +{text} +[END TEXT] + +{format_instructions} diff --git a/evals/prompts/vocabulary/other-grades-system.txt b/evals/prompts/vocabulary/other-grades-system.txt new file mode 100644 index 0000000..8c65f65 --- /dev/null +++ b/evals/prompts/vocabulary/other-grades-system.txt @@ -0,0 +1,9 @@ + +You are an expert curriculum designer. Your job involves reading text snippets intended for students in K-12 and evaluating the complexity of the vocabulary in each text. + +You will be given a rubric (with options 1, 2, 3, 4) as well as guidelines for interpreting the rubric. + +IMPORTANT: You should only pay attention to the vocabulary. Do not evaluate any other element of the text's complexity (e.g. sentence structure, meainng, etc.) +IMPORTANT: Rely on the supplied rubric and annotation guidelines along. Do not introduce any new crtieria for evaluating the complexity of a text's vocabulary. + +Please first reason out loud about the vocabulary complexity of the text and then provide an answer between 1 and 4 (whole numbers only). Provide the answer as an integer (not a float). diff --git a/evals/prompts/vocabulary/other-grades-user.txt b/evals/prompts/vocabulary/other-grades-user.txt new file mode 100644 index 0000000..0d4b534 --- /dev/null +++ b/evals/prompts/vocabulary/other-grades-user.txt @@ -0,0 +1,139 @@ + +Your job is to rate the complexity of a text's vocabulary (relative to the intended level of the text) according to a rubric and annotation guide. Stick to the rubric and annotation guide exactly — do not introduce any additional criteria or lenses for judging the complexity of the text. + +[BEGIN ANNOTATION GUIDE AND RUBRIC] +Instructions +For the following task, please assume that: + - The student is on grade level and proficient in all core content areas, including reading fluency, comprehension, science, & social studies. (example). + - The student is moving through a common progression of topics (detailed here). + - The student is fluent in speaking English. + - The student has an "average" amount of background knowledge on topics not commonly covered in curriculum. + - The student will use this material for independent reading/work, without direct instruction. + - The text is reasonable for the given grade level. + +Please do not consider the presence of figurative language when scoring Vocabulary. For example: with a phrase like "kicked the bucket," consider only the qualities of the words themselves ("kicked", "the" and "bucket"). + +Please do be sure to consider: +- all of the different types of vocabulary (listed below) +- the overall proportion of complex words in the text - including repeated complex words. +- the resulting holistic complexity of the vocabulary (described in the Summary section below). + +Level 1: +Rubric: Vocabulary that is almost entirely not complex: contemporary, conversational, and/or familiar. That said, a very low proportion of complex words (archaic, subject-specific, academic) is OK -- i.e. doesn't need to be 0. + +Level 2: +Rubric: Vocabulary that is mostly not complex: contemporary, conversational, and/or familiar. A low proportion of complex words (archaic, subject-specific, academic) is OK, but if it's very low, the text is probably level 1. + +Level 3: +Rubric: Vocabulary that is often complex: unfamiliar, archaic, subject-specific, and/or overly academic + +Level 4: +Rubric: Vocabulary that is mostly complex: unfamiliar, archaic, subject-specific, and/or overly academic. May be ambiguous or purposefully misleading + +And here are some relevant definitions: + - Conversational: Everyday language. + - Familiar: Words that the student is likely to have seen/heard, from everyday life or their curriculum. Reminder: assume an "average" level of background knowledge. + - Unfamiliar: Words the student has probably not heard, or are being used in an unfamiliar way. + - For ex: 4th graders are familiar with the word "table" but may not be familiar with the use of the word with respect to data ("a table of data"). + - Note: + - Words with in-line definitions (via appositives, or because they can be easily inferred from other parts of the text) should be evaluated as less unfamiliar. + - For ex: "The pharaoh, a powerful ruler of ancient Egypt, was buried in a grand tomb." + - The word "pharaoh" might be unfamiliar or subject-specific, but since is defined within the text, you can consider it a more familiar word. + - Unfamiliar proper nouns: + - A person's name, even if unfamiliar, generally does not add to complexity. + - Other unfamiliar proper nouns (eg locations, organizations) do add to complexity. + +- Subject-specific: Words that are specific to a subject or field of study that are essential for understanding concepts and engaging with the content. +- Overly-academic: Words that are excessively formal, complex, or specialized. + - For ex: "The agrarian societal structure of the Neolithic Revolution precipitated a paradigm shift in agriculture" +- Archaic: A word that was common in the past but is now rarely/almost never used. Could also be a word used in an archaic way. + - For ex: "After a long day of court proceedings, the jury 'retired' to deliberate on their verdict." + - The word "retire" meaning to stop working may be familiar to a student, but "retire" meaning "withdrawing to a private place" is an archaic use. + + +Examples +The student is on-grade-level: +- Consider a 6th grade passage about earth systems. Per NGSS standards, students are introduced to earth systems starting in 2nd grade. They encounter words like: wind, water, river, lake, solids, and liquids. For our rating purposes, we would assume most students following 2nd have encountered these words. In 5th grade, they dive more fully into earth systems concepts, learning vocabulary words like geosphere, sediment, biosphere, atmosphere, ecosystems, organisms and climate. While rating, we would consider the words listed in the NGSS standards as more familiar following that grade level. If the same passage were intended for 3rd graders, though, then the subject-specific vocabulary is likely to be unfamiliar. + +Figurative Language +- Kicked the bucket. +- The pen is mightier than the sword. +- The classroom was a zoo. +- He ran faster than the speed of light. +[END ANNOTATION GUIDE AND RUBRIC] + +Here are a couple examples of texts that have already been scored along with justification for their scores, which you can use as exemplars: +[BEGIN EXAMPLES] + +*** EXAMPLE 1 *** +The following text was intended for grade level 11 and received a complexity level of 1. + +Here is the background knowledge assumption for that text: N/A + +Here is the text: +// START TEXT // +"In a recent lecture, "Is Nothing Sacred?", Salman Rushdie, one of the most censored authors of our time, talked about the importance of books. He grew up in a household in India where books were as sacred as bread. If anyone in the household dropped a piece of bread or a book, the person not only picked it up, but also kissed the object by way of apologizing for clumsy disrespect. + +He goes on to say that he had kissed many books before he had kissed a girl. Bread and books were for his household, and for many like his, food for the body and the soul. This image of the kissing of the book one had accidentally dropped made an impression on me. It speaks to the love and respect many people have for them. + +I grew up in a small town in New Mexico, and we had very few books in our household. The first one I remember reading was my catechism book. Before I went to school to learn English, my mother taught me catechism in Spanish. + +I remember the questions and answers I had to learn, and I remember the well-thumbed, frayed volume which was sacred to me. + +Growing up with few books in the house created in me a desire and a need for them. When I started school, I remember visiting the one room library of our town and standing in front of the dusty shelves. In reality there were only a few shelves and not over a thousand books, but I wanted to read them all. There was food for my soul in the books, that much I realized." +// END TEXT // + +Here is the reasoning for that complexity level: +// START REASONING // +This text is a 1 for vocabulary, because the vocabulary that is used is familiar and accessible for a proficient 11th grader. Most of the words used in the text are very common everyday vocabulary for describing growing up, family life, and the importance of reading. A few examples of these very common words are: small town, book, school, learn, food, kissed, image, respect, love, speaks. There are many more in the text. In this text there are only a few "juicier" or more complex words, you can think of those as words that are less familiar, have a more abstract or nuanced meaning, or carry a very large concept. Less commonly spoken words that were used in the text were: frayed, volume, censored, clumsy, sacred. These are still well within reach of a proficient 11th grader, and would still be considered familiar, because they will have encountered them in past reading or academic studies. In the text there are a couple of words that are outliers, but they are not essential to the understanding of the larger text. One of these words or hyphenated compound phrase is well-frayed. A compound phrase is a phrase consisting of multiple words that work together to create a specific meaning or idea, often acting as a single unit in a sentence. If the meaning of individual words is familiar, it is typically quite easy for proficient readers to generalize the larger meaning that the author is implying with their word choice. In this case, proficient students will be accustomed to the phrase well, with the secondary meaning of very, rather than a description of positivity or health; and they will be accustomed to the use frayed, as in worn, aged, or damaged from use. Making the leap to identify the meaning of "well-frayed" as a book that is very used, will take only moments for a proficient 11th grader. Another word that stands out in the text is the word catechism, which might be new for many students based on their personal background or location, but a full understanding of what a catechism book contains is not essential for understanding the paragraph or whole text. The reader can make it through using minimum context clues to know that the catechism must be something important to his family. The type of book he learned to read before going to school is not critical for comprehension, it's enough to understand that reading was so important in his family, his mother started instruction before he even started school. Additionally, it's important to know that having one unknown word for an 11th grade reading, does not merit a rating higher than one. + +It is worth noting that another reason this text is a 1, is that the content or topic of the passage is so familiar and covered extensively in K-12 education, i.e. reading is important, loving books, growing up; that coupled with the simple vocabulary choices, getting to the meaning of the overall text, and even the paragraphs, would be incredibly easy for a proficient 11th grader. +// END REASONING // +*** EXAMPLE 2 *** +The following text was intended for grade level 5 and received a complexity level of 2. + +Here is the background knowledge assumption for that text: Background Knowledge Assumption: Students are likely familiar with the concept of natural disasters, including hurricanes, and basic atmospheric concepts like high and low pressure from their studies on weather and climate. They may not be familiar with the specific formation processes of hurricanes or the global terminology differences (hurricane, typhoon, cyclone). + +Here is the text: +// START TEXT // +Great whirling storms roar out of the oceans in many parts of the world. They are called by several names—hurricane, typhoon, and cyclone are the three most familiar ones. But no matter what they are called, they are all the same sort of storm. They are born in the same way, in tropical waters. They develop the same way, feeding on warm, moist air. And they do the same kind of damage, both ashore and at sea. Other storms may cover a bigger area or have higher winds, but none can match both the size and the fury of hurricanes. They are earth's mightiest storms. + +Like all storms, they take place in the atmosphere, the envelope of air that surrounds the earth and presses on its surface. The pressure at any one place is always changing. There are days when air is sinking and the atmosphere presses harder on the surface. These are the times of high pressure. There are days when a lot of air is rising and the atmosphere does not press down as hard. These are times of low pressure. Low-pressure areas over warm oceans give birth to hurricanes. +// END TEXT // + +Here is the reasoning for that complexity level: +// START REASONING // +I scored this a 2 because of the density of subject-specific vocabulary related to weather and climate, which is often covered in lower grade levels. This adds to the complexity above a 1, but it is not a level 3 because of the familiarity with the topic, which implies some familiarity with the vocabulary as well. The specific formation process and the vocabulary used to explain the processes are also subject-specfiic but not famliar, which would make the second paragraph a level 3 in the rubric language, but when considering the language used in the overall SUMMARY below the rubric, this new content and vocabulary would cause quick pauses and/or occasional prolonged pauses but would not cause the reader to slow down to due to challenging overall comprehension of the key ideas and supporting claims. This is especially the case because the second paragraph builds upon prior knowledge and familiar vocabulary use, so it is not entirely new information and vocabulary. While there is subject-specific vocabulary used, overly academic vocabulary is NOT used and is more conversational in nature, such as "great whiring storms" and "born" / "giving birth" to storm (although this is the way storms are described!) rather than more technical terms which made comprehension easier due to the accessibility of the vocabulary (even if used in other contexts before reading this text). Words such as "a lot" and "bigger" are more conversational, and while technical, unfamiliar words are provided, such as "hurricane," "typhoon," and "cyclone," knowing and understanding their differences is not necessary to grasp the main idea. The processes by which they are formed are what need to be retained while reading the entire text, and familiarity with the bulk of the vocabulary used would allow for that to happen without too much struggle to make meaning of it. Additionally, the text does not contain any archaic vocabulary or ambiguous words, which prevents it from reaching a rating of 4, although it is not necessary that they text have such vocabulary to meet a level 4, the frequent inclusion of such vocabulary makes it more likely to land at least a 3 or 4. +// END REASONING // + +*** EXAMPLE 3 *** +The following text was intended for grade level 6 and received a complexity level of 3. + +Here is the background knowledge assumption for that text: Background Knowledge Assumption: Students are likely familiar with basic Earth science concepts such as rocks, minerals, and fossils, as well as natural processes like volcanic eruptions and earthquakes. They may not be familiar with more advanced topics like plate tectonics or the specific branches of geology such as mineralogy, petrology, and seismology. + +Here is the text: +// START TEXT // +Geology is the scientific study of Earth. Geologists study the planet—its formation, its internal structure, its materials, its chemical and physical processes, and its history. Mountains, valleys, plains, sea floors, minerals, rocks, fossils, and the processes that create and destroy each of these are all the domain of the geologist. Geology is divided into two broad categories of study: physical geology and historical geology. + +Physical geology is concerned with the processes occurring on or below the surface of Earth and the materials on which they operate. These processes include volcanic eruptions, landslides, earthquakes, and floods. Materials include rocks, air, seawater, soils, and sediment. Physical geology further divides into more specific branches, each of which deals with its own part of Earth's materials, landforms, and processes. Mineralogy and petrology investigate the composition and origin of minerals and rocks. Volcanologists study lava, rocks, and gases on live, dormant, and extinct volcanoes. Seismologists use instruments to monitor and predict earthquakes and volcanic eruptions. + +Historical geology is concerned with the chronology of events, both physical and biological, that have taken place in Earth's history. Paleontologists study fossils (remains of ancient life) for evidence of the evolution of life on Earth. Fossils not only relate evolution, but also speak of the environment in which the organism lived. Corals in rocks at the top of the Grand Canyon in Arizona, for example, show a shallow sea flooded the area around 290 million years ago. In addition, by determining the ages and types of rocks around the world, geologists piece together continental and oceanic history over the past few billion years. Plate tectonics (the study of the movement of the sections of Earth's crust) adds to Earth's story with details of the changing configuration of the continents and oceans. +// END TEXT // + +Here is the reasoning for that complexity level: +// START REASONING // +To determine the complexity rating of this text based on the vocabulary present, I used the annotation guide, scoring rubric, and examples to set the expectations for rating. During the first read of the text, I "bolded" and categorized the more challenging vocabulary words according to the following complexity groupings: archaic, unfamiliar, archaic, subject-specific, and/or overly academic. On the second read, I considered the main idea or "gist" that students need to acquire understanding of. I then referenced the previously mentioned tools–annotation guide, scoring rubric, and examples to remind myself of the expectations for rating. I agreed that readers would have familiarity with basic concepts of geology; however, I also considered the definitions provided for words such as Geology, Geologists, Physical Geology, Historical Geology, Mineralogy, and Petrology. I considered how students might pause for clarification and for how long. After reviewing the Annotation Guide while considering, I narrowed the rating down because the definitions provided throughout the text of more complex words should make the meaning of the text more accessible for readers, which is why although the words are subject-specific, I rated this text as a 3 instead of a 2-less complex or a 4–more complex. I read the text one final time to ensure clarity around my rating, scored and wrote the justification. +// END REASONING // +[END EXAMPLES] + +Below is the text you need to evaluate. It is intended for grade {student_grade_level}. + +As you read the text, you can assume the student has the following background knowledge about the text — this background knowledge influences which words from the text are familiar versus unfamiliar for the student: {student_background_knowledge} + +[BEGIN TEXT] +{text} +[END TEXT] + +In your response, when specifying the level of complexity, be sure to use only a single integer (e.g. 2) and don't include any other text (e.g. don't say "level 2"). + +{format_instructions} From cad8ca4cb40b26e4b0c3885c3b12843b50d6d9e2 Mon Sep 17 00:00:00 2001 From: Adnan Hussain <61515575+adnanrhussain@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:20:23 -0800 Subject: [PATCH 2/9] chore: Setup sdk infra (#11) * chore: Setup sdk infra --- .github/workflows/test-sdk-typescript.yml | 173 + sdks/typescript/.env.test.example | 20 + sdks/typescript/.eslintrc.cjs | 23 + sdks/typescript/.gitignore | 37 + sdks/typescript/.nvmrc | 1 + sdks/typescript/LICENSE | 12 + sdks/typescript/README.md | 1 + sdks/typescript/package-lock.json | 3861 +++++++++++++++++++++ sdks/typescript/package.json | 75 + sdks/typescript/src/index.ts | 1 + sdks/typescript/src/types/txt.d.ts | 4 + sdks/typescript/tsconfig.json | 28 + sdks/typescript/tsup.config.ts | 16 + sdks/typescript/vitest.ci.config.ts | 34 + sdks/typescript/vitest.config.ts | 39 + 15 files changed, 4325 insertions(+) create mode 100644 .github/workflows/test-sdk-typescript.yml create mode 100644 sdks/typescript/.env.test.example create mode 100644 sdks/typescript/.eslintrc.cjs create mode 100644 sdks/typescript/.gitignore create mode 100644 sdks/typescript/.nvmrc create mode 100644 sdks/typescript/LICENSE create mode 100644 sdks/typescript/README.md create mode 100644 sdks/typescript/package-lock.json create mode 100644 sdks/typescript/package.json create mode 100644 sdks/typescript/src/index.ts create mode 100644 sdks/typescript/src/types/txt.d.ts create mode 100644 sdks/typescript/tsconfig.json create mode 100644 sdks/typescript/tsup.config.ts create mode 100644 sdks/typescript/vitest.ci.config.ts create mode 100644 sdks/typescript/vitest.config.ts diff --git a/.github/workflows/test-sdk-typescript.yml b/.github/workflows/test-sdk-typescript.yml new file mode 100644 index 0000000..6af11d2 --- /dev/null +++ b/.github/workflows/test-sdk-typescript.yml @@ -0,0 +1,173 @@ +name: 🏗️ Test TypeScript SDK + +on: + push: + branches: + - main + paths: + - 'sdks/typescript/**' + - 'evals/prompts/**' + - '.github/workflows/test-sdk-typescript.yml' + pull_request: + paths: + - 'sdks/typescript/**' + - 'evals/prompts/**' + - '.github/workflows/test-sdk-typescript.yml' + +jobs: + lint: + name: 👖 Lint + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdks/typescript + steps: + - name: ⛔ Cancel previous runs + uses: styfle/cancel-workflow-action@0.13.0 + + - name: ⬇️ Checkout repo + uses: actions/checkout@v6 + + - name: 😻 Setup Node.js 22 + uses: actions/setup-node@v6 + with: + node-version: 22 + + - name: 📥 Install dependencies + uses: bahmutov/npm-install@v1 + with: + working-directory: sdks/typescript + + - name: 👖 Run linter + run: npm run lint + + typecheck: + name: 🔎 TypeScript + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdks/typescript + steps: + - name: ⛔ Cancel previous runs + uses: styfle/cancel-workflow-action@0.13.0 + + - name: ⬇️ Checkout repo + uses: actions/checkout@v6 + + - name: 😻 Setup Node.js 22 + uses: actions/setup-node@v6 + with: + node-version: 22 + + - name: 📥 Install dependencies + uses: bahmutov/npm-install@v1 + with: + working-directory: sdks/typescript + + - name: 🔎 Type check + run: npm run typecheck + + test: + name: ⚡ Unit Tests (Node ${{ matrix.node-version }}) + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdks/typescript + strategy: + matrix: + node-version: ['20.19.0', '22', '24'] + steps: + - name: ⛔ Cancel previous runs + uses: styfle/cancel-workflow-action@0.13.0 + + - name: ⬇️ Checkout repo + uses: actions/checkout@v6 + + - name: 😻 Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v6 + with: + node-version: ${{ matrix.node-version }} + + - name: 📥 Install dependencies + uses: bahmutov/npm-install@v1 + with: + working-directory: sdks/typescript + + - name: ⚡ Run unit tests + run: npm run test:unit + + build: + name: 🏗️ Build (Node ${{ matrix.node-version }}) + needs: [lint, typecheck, test] + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdks/typescript + strategy: + matrix: + node-version: ['20.19.0', '22', '24'] + steps: + - name: ⛔ Cancel previous runs + uses: styfle/cancel-workflow-action@0.13.0 + + - name: ⬇️ Checkout repo + uses: actions/checkout@v6 + + - name: 😻 Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v6 + with: + node-version: ${{ matrix.node-version }} + + - name: 📥 Install dependencies + uses: bahmutov/npm-install@v1 + with: + working-directory: sdks/typescript + + - name: 🏗️ Build package + run: npm run build + + # Temporarily disabled - integration tests take too long in CI + # - name: ⚡ Run integration tests (against dist/) + # if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + # env: + # RUN_INTEGRATION_TESTS: true + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + # run: npm run test:integration:dist + + coverage: + name: 📊 Coverage + needs: [build] + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdks/typescript + steps: + - name: ⛔ Cancel previous runs + uses: styfle/cancel-workflow-action@0.13.0 + + - name: ⬇️ Checkout repo + uses: actions/checkout@v6 + + - name: 😻 Setup Node.js 22 + uses: actions/setup-node@v6 + with: + node-version: 22 + + - name: 📥 Install dependencies + uses: bahmutov/npm-install@v1 + with: + working-directory: sdks/typescript + + - name: 📊 Generate coverage report + run: npm run test:coverage + continue-on-error: true + + - name: 📁 Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./sdks/typescript/coverage/coverage-final.json + flags: typescript-sdk + name: typescript-sdk-coverage + continue-on-error: true diff --git a/sdks/typescript/.env.test.example b/sdks/typescript/.env.test.example new file mode 100644 index 0000000..c882775 --- /dev/null +++ b/sdks/typescript/.env.test.example @@ -0,0 +1,20 @@ +# ============================================================================= +# FOR SDK CONTRIBUTORS ONLY — Running integration tests locally +# ============================================================================= +# This file is NOT needed to use the SDK. The SDK requires all configuration +# (model, API keys, etc.) to be passed in explicitly by the consumer. +# +# To run integration tests locally: +# 1. Copy this file to .env.test.local +# 2. Fill in the API keys below +# 3. Run: npm run test:integration +# ============================================================================= + +# OpenAI API Key — https://platform.openai.com/api-keys +# OPENAI_API_KEY= + +# Google API Key (for Gemini) — https://makersuite.google.com/app/apikey +# GOOGLE_API_KEY= + +# Anthropic API Key — https://console.anthropic.com/ +# ANTHROPIC_API_KEY= diff --git a/sdks/typescript/.eslintrc.cjs b/sdks/typescript/.eslintrc.cjs new file mode 100644 index 0000000..0c24508 --- /dev/null +++ b/sdks/typescript/.eslintrc.cjs @@ -0,0 +1,23 @@ +module.exports = { + parser: '@typescript-eslint/parser', + parserOptions: { + ecmaVersion: 2022, + sourceType: 'module', + project: './tsconfig.json', + tsconfigRootDir: __dirname, + }, + plugins: ['@typescript-eslint'], + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + ], + rules: { + '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }], + '@typescript-eslint/no-explicit-any': 'warn', + '@typescript-eslint/explicit-module-boundary-types': 'off', + }, + env: { + node: true, + es2022: true, + }, +}; diff --git a/sdks/typescript/.gitignore b/sdks/typescript/.gitignore new file mode 100644 index 0000000..e02ce86 --- /dev/null +++ b/sdks/typescript/.gitignore @@ -0,0 +1,37 @@ +# Dependencies +node_modules/ +.pnp +.pnp.js + +# Build output +dist/ +build/ +*.tsbuildinfo + +# Testing +coverage/ +.nyc_output/ + +# Environment variables +.env +.env.local +.env.*.local + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Logs +logs/ +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* diff --git a/sdks/typescript/.nvmrc b/sdks/typescript/.nvmrc new file mode 100644 index 0000000..2bd5a0a --- /dev/null +++ b/sdks/typescript/.nvmrc @@ -0,0 +1 @@ +22 diff --git a/sdks/typescript/LICENSE b/sdks/typescript/LICENSE new file mode 100644 index 0000000..e8c406f --- /dev/null +++ b/sdks/typescript/LICENSE @@ -0,0 +1,12 @@ +The Evaluator code is licensed under [MIT](https://opensource.org/license/mit). + +Evaluators content (including the prompt and settings information) is provided by Learning Commons under the CC BY 4.0 International license ([CC BY 4.0](https://creativecommons.org/licenses/by/4.0/deed.en)). + +Annotated CLEAR Corpus is provided by Learning Commons (including annotations and enhancements) under CC BY-NC-SA 4.0 ([CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/deed.en)). The original dataset from CLEAR Corpus can be found at [The CLEAR Corpus by CommonLit](https://www.commonlit.org/blog/introducing-the-clear-corpus-an-open-dataset-to-advance-research-28ff8cfea84a/) licensed under CC BY-NC-SA 4.0. + +**How to Cite the Evaluator Code:** +Learning Commons (2025). Evaluators. GitHub. [https://github.com/learning-commons-org/evaluators](https://github.com/learning-commons-org/evaluators). +Licensed under MIT. + +**How to Cite the Evaluator:** +Learning Commons. (2025). Evaluators content (including the prompt and settings information) is available at GitHub. [https://github.com/learning-commons-org/evaluators](https://github.com/learning-commons-org/evaluators) Licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/deed.en) diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md new file mode 100644 index 0000000..3ea9d1c --- /dev/null +++ b/sdks/typescript/README.md @@ -0,0 +1 @@ +# @learning-commons/evaluators diff --git a/sdks/typescript/package-lock.json b/sdks/typescript/package-lock.json new file mode 100644 index 0000000..e3c2e86 --- /dev/null +++ b/sdks/typescript/package-lock.json @@ -0,0 +1,3861 @@ +{ + "name": "@learning-commons/evaluators", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@learning-commons/evaluators", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "compromise": "^14.13.0", + "syllable": "^5.0.1", + "zod": "^3.22.4" + }, + "devDependencies": { + "@ai-sdk/anthropic": "^3.0.12", + "@ai-sdk/google": "^3.0.7", + "@ai-sdk/openai": "^3.0.9", + "@types/node": "^20.11.5", + "@typescript-eslint/eslint-plugin": "^6.19.0", + "@typescript-eslint/parser": "^6.19.0", + "@vitest/coverage-v8": "^4.0.17", + "ai": "^6.0.30", + "eslint": "^8.56.0", + "tsup": "^8.0.1", + "typescript": "^5.3.3", + "vitest": "^4.0.17" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "ai": ">=4.0.0" + } + }, + "node_modules/@ai-sdk/anthropic": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.12.tgz", + "integrity": "sha512-1ygMQ2acJ9k+CPRuDiywtkuQwSdsZwAISgbRIayE3PF/Ct2b80c4iGS1CbycYMx+yswPyBMOuPTqebh9vvwMMg==", + "dev": true, + "dependencies": { + "@ai-sdk/provider": "3.0.2", + "@ai-sdk/provider-utils": "4.0.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.12.tgz", + "integrity": "sha512-lL2tSNOQBu3rMcfF59jtpxGH3uF0CVOsoBWp/9Qi65S8umQQXxgx9KH3NXcFDdu0APZPXVww4Z757l/BA2ysSw==", + "dev": true, + "dependencies": { + "@ai-sdk/provider": "3.0.2", + "@ai-sdk/provider-utils": "4.0.5", + "@vercel/oidc": "3.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/google": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.7.tgz", + "integrity": "sha512-GrmToTWMFJXma6DIEjbHBzTuAAiwkWg7CDIbrjKmUp7T8BiTo4ndNSs08xspe13Ast5Gx2FvLCHH0MYJM2RpOg==", + "dev": true, + "dependencies": { + "@ai-sdk/provider": "3.0.2", + "@ai-sdk/provider-utils": "4.0.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.9.tgz", + "integrity": "sha512-azgo1gmAFwkCDHKWlv9goKBe7SOG5c8zxIX94SEf8748t+ZL0sjPH2RNXk7G6POaZ4A6Os4zhkUnx9KwSk9Bjw==", + "dev": true, + "dependencies": { + "@ai-sdk/provider": "3.0.2", + "@ai-sdk/provider-utils": "4.0.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.2.tgz", + "integrity": "sha512-HrEmNt/BH/hkQ7zpi2o6N3k1ZR1QTb7z85WYhYygiTxOQuaml4CMtHCWRbric5WPU+RNsYI7r1EpyVQMKO1pYw==", + "dev": true, + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.5.tgz", + "integrity": "sha512-Ow/X/SEkeExTTc1x+nYLB9ZHK2WUId8+9TlkamAx7Tl9vxU+cKzWx2dwjgMHeCN6twrgwkLrrtqckQeO4mxgVA==", + "dev": true, + "dependencies": { + "@ai-sdk/provider": "3.0.2", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.6.tgz", + "integrity": "sha512-TeR9zWR18BvbfPmGbLampPMW+uW1NZnJlRuuHso8i87QZNq2JRF9i6RgxRqtEq+wQGsS19NNTWr2duhnE49mfQ==", + "dev": true, + "dependencies": { + "@babel/types": "^7.28.6" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.6.tgz", + "integrity": "sha512-0ZrskXVEHSWIqZM/sQZ4EV3jZJXRkio/WCxaqKZP1g//CEWEPSfeZFcms4XeKBCHU0ZKnIkdJeU/kF+eRp5lBg==", + "dev": true, + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", + "integrity": "sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.2.tgz", + "integrity": "sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.2.tgz", + "integrity": "sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.2.tgz", + "integrity": "sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.2.tgz", + "integrity": "sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.2.tgz", + "integrity": "sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.2.tgz", + "integrity": "sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.2.tgz", + "integrity": "sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.2.tgz", + "integrity": "sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.2.tgz", + "integrity": "sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.2.tgz", + "integrity": "sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.2.tgz", + "integrity": "sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.2.tgz", + "integrity": "sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.2.tgz", + "integrity": "sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.2.tgz", + "integrity": "sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.2.tgz", + "integrity": "sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.2.tgz", + "integrity": "sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.2.tgz", + "integrity": "sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.2.tgz", + "integrity": "sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.2.tgz", + "integrity": "sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.2.tgz", + "integrity": "sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.2.tgz", + "integrity": "sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.2.tgz", + "integrity": "sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.2.tgz", + "integrity": "sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.2.tgz", + "integrity": "sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.2.tgz", + "integrity": "sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@eslint/eslintrc/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@eslint/js": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", + "deprecated": "Use @eslint/config-array instead", + "dev": true, + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.3", + "debug": "^4.3.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@humanwhocodes/config-array/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", + "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", + "dev": true + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.55.1.tgz", + "integrity": "sha512-9R0DM/ykwfGIlNu6+2U09ga0WXeZ9MRC2Ter8jnz8415VbuIykVuc6bhdrbORFZANDmTDvq26mJrEVTl8TdnDg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.55.1.tgz", + "integrity": "sha512-eFZCb1YUqhTysgW3sj/55du5cG57S7UTNtdMjCW7LwVcj3dTTcowCsC8p7uBdzKsZYa8J7IDE8lhMI+HX1vQvg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.55.1.tgz", + "integrity": "sha512-p3grE2PHcQm2e8PSGZdzIhCKbMCw/xi9XvMPErPhwO17vxtvCN5FEA2mSLgmKlCjHGMQTP6phuQTYWUnKewwGg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.55.1.tgz", + "integrity": "sha512-rDUjG25C9qoTm+e02Esi+aqTKSBYwVTaoS1wxcN47/Luqef57Vgp96xNANwt5npq9GDxsH7kXxNkJVEsWEOEaQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.55.1.tgz", + "integrity": "sha512-+JiU7Jbp5cdxekIgdte0jfcu5oqw4GCKr6i3PJTlXTCU5H5Fvtkpbs4XJHRmWNXF+hKmn4v7ogI5OQPaupJgOg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.55.1.tgz", + "integrity": "sha512-V5xC1tOVWtLLmr3YUk2f6EJK4qksksOYiz/TCsFHu/R+woubcLWdC9nZQmwjOAbmExBIVKsm1/wKmEy4z4u4Bw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.55.1.tgz", + "integrity": "sha512-Rn3n+FUk2J5VWx+ywrG/HGPTD9jXNbicRtTM11e/uorplArnXZYsVifnPPqNNP5BsO3roI4n8332ukpY/zN7rQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.55.1.tgz", + "integrity": "sha512-grPNWydeKtc1aEdrJDWk4opD7nFtQbMmV7769hiAaYyUKCT1faPRm2av8CX1YJsZ4TLAZcg9gTR1KvEzoLjXkg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.55.1.tgz", + "integrity": "sha512-a59mwd1k6x8tXKcUxSyISiquLwB5pX+fJW9TkWU46lCqD/GRDe9uDN31jrMmVP3feI3mhAdvcCClhV8V5MhJFQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.55.1.tgz", + "integrity": "sha512-puS1MEgWX5GsHSoiAsF0TYrpomdvkaXm0CofIMG5uVkP6IBV+ZO9xhC5YEN49nsgYo1DuuMquF9+7EDBVYu4uA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.55.1.tgz", + "integrity": "sha512-r3Wv40in+lTsULSb6nnoudVbARdOwb2u5fpeoOAZjFLznp6tDU8kd+GTHmJoqZ9lt6/Sys33KdIHUaQihFcu7g==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.55.1.tgz", + "integrity": "sha512-MR8c0+UxAlB22Fq4R+aQSPBayvYa3+9DrwG/i1TKQXFYEaoW3B5b/rkSRIypcZDdWjWnpcvxbNaAJDcSbJU3Lw==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.55.1.tgz", + "integrity": "sha512-3KhoECe1BRlSYpMTeVrD4sh2Pw2xgt4jzNSZIIPLFEsnQn9gAnZagW9+VqDqAHgm1Xc77LzJOo2LdigS5qZ+gw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.55.1.tgz", + "integrity": "sha512-ziR1OuZx0vdYZZ30vueNZTg73alF59DicYrPViG0NEgDVN8/Jl87zkAPu4u6VjZST2llgEUjaiNl9JM6HH1Vdw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.55.1.tgz", + "integrity": "sha512-uW0Y12ih2XJRERZ4jAfKamTyIHVMPQnTZcQjme2HMVDAHY4amf5u414OqNYC+x+LzRdRcnIG1YodLrrtA8xsxw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.55.1.tgz", + "integrity": "sha512-u9yZ0jUkOED1BFrqu3BwMQoixvGHGZ+JhJNkNKY/hyoEgOwlqKb62qu+7UjbPSHYjiVy8kKJHvXKv5coH4wDeg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.55.1.tgz", + "integrity": "sha512-/0PenBCmqM4ZUd0190j7J0UsQ/1nsi735iPRakO8iPciE7BQ495Y6msPzaOmvx0/pn+eJVVlZrNrSh4WSYLxNg==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.55.1.tgz", + "integrity": "sha512-a8G4wiQxQG2BAvo+gU6XrReRRqj+pLS2NGXKm8io19goR+K8lw269eTrPkSdDTALwMmJp4th2Uh0D8J9bEV1vg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.55.1.tgz", + "integrity": "sha512-bD+zjpFrMpP/hqkfEcnjXWHMw5BIghGisOKPj+2NaNDuVT+8Ds4mPf3XcPHuat1tz89WRL+1wbcxKY3WSbiT7w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.55.1.tgz", + "integrity": "sha512-eLXw0dOiqE4QmvikfQ6yjgkg/xDM+MdU9YJuP4ySTibXU0oAvnEWXt7UDJmD4UkYialMfOGFPJnIHSe/kdzPxg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.55.1.tgz", + "integrity": "sha512-xzm44KgEP11te3S2HCSyYf5zIzWmx3n8HDCc7EE59+lTcswEWNpvMLfd9uJvVX8LCg9QWG67Xt75AuHn4vgsXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.55.1.tgz", + "integrity": "sha512-yR6Bl3tMC/gBok5cz/Qi0xYnVbIxGx5Fcf/ca0eB6/6JwOY+SRUcJfI0OpeTpPls7f194as62thCt/2BjxYN8g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.55.1.tgz", + "integrity": "sha512-3fZBidchE0eY0oFZBnekYCfg+5wAB0mbpCBuofh5mZuzIU/4jIVkbESmd2dOsFNS78b53CYv3OAtwqkZZmU5nA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.55.1.tgz", + "integrity": "sha512-xGGY5pXj69IxKb4yv/POoocPy/qmEGhimy/FoTpTSVju3FYXUQQMFCaZZXJVidsmGxRioZAwpThl/4zX41gRKg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.55.1.tgz", + "integrity": "sha512-SPEpaL6DX4rmcXtnhdrQYgzQ5W2uW3SCJch88lB2zImhJRhIIK44fkUrgIV/Q8yUNfw5oyZ5vkeQsZLhCb06lw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true + }, + "node_modules/@types/node": { + "version": "20.19.28", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.28.tgz", + "integrity": "sha512-VyKBr25BuFDzBFCK5sUM6ZXiWfqgCTwTAOK8qzGV/m9FCirXYDlmczJ+d5dXBAQALGCdRRdbteKYfJ84NGEusw==", + "dev": true, + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/pluralize": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/pluralize/-/pluralize-0.0.29.tgz", + "integrity": "sha512-BYOID+l2Aco2nBik+iYS4SZX0Lf20KPILP5RGmM1IgzdwNdTs0eebiFriOPcej1sX9mLnSoiNte5zcFxssgpGA==" + }, + "node_modules/@types/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==", + "dev": true + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.21.0.tgz", + "integrity": "sha512-oy9+hTPCUFpngkEZUSzbf9MxI65wbKFoQYsgPdILTfbUldp5ovUuphZVe4i30emU9M/kP+T64Di0mxl7dSw3MA==", + "dev": true, + "dependencies": { + "@eslint-community/regexpp": "^4.5.1", + "@typescript-eslint/scope-manager": "6.21.0", + "@typescript-eslint/type-utils": "6.21.0", + "@typescript-eslint/utils": "6.21.0", + "@typescript-eslint/visitor-keys": "6.21.0", + "debug": "^4.3.4", + "graphemer": "^1.4.0", + "ignore": "^5.2.4", + "natural-compare": "^1.4.0", + "semver": "^7.5.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^6.0.0 || ^6.0.0-alpha", + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.21.0.tgz", + "integrity": "sha512-tbsV1jPne5CkFQCgPBcDOt30ItF7aJoZL997JSF7MhGQqOeT3svWRYxiqlfA5RUdlHN6Fi+EI9bxqbdyAUZjYQ==", + "dev": true, + "dependencies": { + "@typescript-eslint/scope-manager": "6.21.0", + "@typescript-eslint/types": "6.21.0", + "@typescript-eslint/typescript-estree": "6.21.0", + "@typescript-eslint/visitor-keys": "6.21.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.21.0.tgz", + "integrity": "sha512-OwLUIWZJry80O99zvqXVEioyniJMa+d2GrqpUTqi5/v5D5rOrppJVBPa0yKCblcigC0/aYAzxxqQ1B+DS2RYsg==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.21.0", + "@typescript-eslint/visitor-keys": "6.21.0" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.21.0.tgz", + "integrity": "sha512-rZQI7wHfao8qMX3Rd3xqeYSMCL3SoiSQLBATSiVKARdFGCYSRvmViieZjqc58jKgs8Y8i9YvVVhRbHSTA4VBag==", + "dev": true, + "dependencies": { + "@typescript-eslint/typescript-estree": "6.21.0", + "@typescript-eslint/utils": "6.21.0", + "debug": "^4.3.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.21.0.tgz", + "integrity": "sha512-1kFmZ1rOm5epu9NZEZm1kckCDGj5UJEf7P1kliH4LKu/RkwpsfqqGmY2OOcUs18lSlQBKLDYBOGxRVtrMN5lpg==", + "dev": true, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.21.0.tgz", + "integrity": "sha512-6npJTkZcO+y2/kr+z0hc4HwNfrrP4kNYh57ek7yCNlrBjWQ1Y0OS7jiZTkgumrvkX5HkEKXFZkkdFNkaW2wmUQ==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.21.0", + "@typescript-eslint/visitor-keys": "6.21.0", + "debug": "^4.3.4", + "globby": "^11.1.0", + "is-glob": "^4.0.3", + "minimatch": "9.0.3", + "semver": "^7.5.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.21.0.tgz", + "integrity": "sha512-NfWVaC8HP9T8cbKQxHcsJBY5YE1O33+jpMwN45qzWWaPDZgLIbo12toGMWnmhvCpd3sIxkpDw3Wv1B3dYrbDQQ==", + "dev": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.4.0", + "@types/json-schema": "^7.0.12", + "@types/semver": "^7.5.0", + "@typescript-eslint/scope-manager": "6.21.0", + "@typescript-eslint/types": "6.21.0", + "@typescript-eslint/typescript-estree": "6.21.0", + "semver": "^7.5.4" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.21.0.tgz", + "integrity": "sha512-JJtkDduxLi9bivAB+cYOVMtbkqdPOhZ+ZI5LC47MIRrDV4Yn2o+ZnW10Nkmr28xRpSpdJ6Sm42Hjf2+REYXm0A==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.21.0", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "dev": true + }, + "node_modules/@vercel/oidc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", + "integrity": "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==", + "dev": true, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@vitest/coverage-v8": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.17.tgz", + "integrity": "sha512-/6zU2FLGg0jsd+ePZcwHRy3+WpNTBBhDY56P4JTRqUN/Dp6CvOEa9HrikcQ4KfV2b2kAHUFB4dl1SuocWXSFEw==", + "dev": true, + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.0.17", + "ast-v8-to-istanbul": "^0.3.10", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.1", + "obug": "^2.1.1", + "std-env": "^3.10.0", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.0.17", + "vitest": "4.0.17" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, + "node_modules/@vitest/expect": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.17.tgz", + "integrity": "sha512-mEoqP3RqhKlbmUmntNDDCJeTDavDR+fVYkSOw8qRwJFaW/0/5zA9zFeTrHqNtcmwh6j26yMmwx2PqUDPzt5ZAQ==", + "dev": true, + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.0.17", + "@vitest/utils": "4.0.17", + "chai": "^6.2.1", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.17.tgz", + "integrity": "sha512-+ZtQhLA3lDh1tI2wxe3yMsGzbp7uuJSWBM1iTIKCbppWTSBN09PUC+L+fyNlQApQoR+Ps8twt2pbSSXg2fQVEQ==", + "dev": true, + "dependencies": { + "@vitest/spy": "4.0.17", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.0.17.tgz", + "integrity": "sha512-Ah3VAYmjcEdHg6+MwFE17qyLqBHZ+ni2ScKCiW2XrlSBV4H3Z7vYfPfz7CWQ33gyu76oc0Ai36+kgLU3rfF4nw==", + "dev": true, + "dependencies": { + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.0.17.tgz", + "integrity": "sha512-JmuQyf8aMWoo/LmNFppdpkfRVHJcsgzkbCA+/Bk7VfNH7RE6Ut2qxegeyx2j3ojtJtKIbIGy3h+KxGfYfk28YQ==", + "dev": true, + "dependencies": { + "@vitest/utils": "4.0.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.0.17.tgz", + "integrity": "sha512-npPelD7oyL+YQM2gbIYvlavlMVWUfNNGZPcu0aEUQXt7FXTuqhmgiYupPnAanhKvyP6Srs2pIbWo30K0RbDtRQ==", + "dev": true, + "dependencies": { + "@vitest/pretty-format": "4.0.17", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.0.17.tgz", + "integrity": "sha512-I1bQo8QaP6tZlTomQNWKJE6ym4SHf3oLS7ceNjozxxgzavRAgZDc06T7kD8gb9bXKEgcLNt00Z+kZO6KaJ62Ew==", + "dev": true, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.17.tgz", + "integrity": "sha512-RG6iy+IzQpa9SB8HAFHJ9Y+pTzI+h8553MrciN9eC6TFBErqrQaTas4vG+MVj8S4uKk8uTT2p0vgZPnTdxd96w==", + "dev": true, + "dependencies": { + "@vitest/pretty-format": "4.0.17", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ai": { + "version": "6.0.30", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.30.tgz", + "integrity": "sha512-66FVOxNTogAkOK3Xx6vR+9l1Ze1bamQl6qmeRONpReqFEBnFInCEFX1EcJVW++BBtfYJ1pPWl5RTDk3BOyCN8w==", + "dev": true, + "dependencies": { + "@ai-sdk/gateway": "3.0.12", + "@ai-sdk/provider": "3.0.2", + "@ai-sdk/provider-utils": "4.0.5", + "@opentelemetry/api": "1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/ast-v8-to-istanbul": { + "version": "0.3.10", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.10.tgz", + "integrity": "sha512-p4K7vMz2ZSk3wN8l5o3y2bJAoZXT3VuJI5OLTATY/01CYWumWvwkUw0SqDBnNq6IiTO3qDa1eSQDibAV8g7XOQ==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^9.0.1" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/bundle-require": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.1.0.tgz", + "integrity": "sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==", + "dev": true, + "dependencies": { + "load-tsconfig": "^0.2.3" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "peerDependencies": { + "esbuild": ">=0.18" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/compromise": { + "version": "14.14.5", + "resolved": "https://registry.npmjs.org/compromise/-/compromise-14.14.5.tgz", + "integrity": "sha512-9qWxpOWo4crzvbdxAYDTwO6z0WljXwi6mL7CqCjAXKn7QtFijmSj7fCyAqGWldCVT2zNboMvg4kNL06drMg2Vw==", + "dependencies": { + "efrt": "2.7.0", + "grad-school": "0.0.5", + "suffix-thumb": "5.0.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/confbox": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", + "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", + "dev": true + }, + "node_modules/consola": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", + "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", + "dev": true, + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/efrt": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/efrt/-/efrt-2.7.0.tgz", + "integrity": "sha512-/RInbCy1d4P6Zdfa+TMVsf/ufZVotat5hCw3QXmWtjU+3pFEOvOQ7ibo3aIxyCJw2leIeAMjmPj+1SLJiCpdrQ==", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz", + "integrity": "sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.2", + "@esbuild/android-arm": "0.27.2", + "@esbuild/android-arm64": "0.27.2", + "@esbuild/android-x64": "0.27.2", + "@esbuild/darwin-arm64": "0.27.2", + "@esbuild/darwin-x64": "0.27.2", + "@esbuild/freebsd-arm64": "0.27.2", + "@esbuild/freebsd-x64": "0.27.2", + "@esbuild/linux-arm": "0.27.2", + "@esbuild/linux-arm64": "0.27.2", + "@esbuild/linux-ia32": "0.27.2", + "@esbuild/linux-loong64": "0.27.2", + "@esbuild/linux-mips64el": "0.27.2", + "@esbuild/linux-ppc64": "0.27.2", + "@esbuild/linux-riscv64": "0.27.2", + "@esbuild/linux-s390x": "0.27.2", + "@esbuild/linux-x64": "0.27.2", + "@esbuild/netbsd-arm64": "0.27.2", + "@esbuild/netbsd-x64": "0.27.2", + "@esbuild/openbsd-arm64": "0.27.2", + "@esbuild/openbsd-x64": "0.27.2", + "@esbuild/openharmony-arm64": "0.27.2", + "@esbuild/sunos-x64": "0.27.2", + "@esbuild/win32-arm64": "0.27.2", + "@esbuild/win32-ia32": "0.27.2", + "@esbuild/win32-x64": "0.27.2" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "dev": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/eslint/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dev": true, + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "dev": true, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fix-dts-default-cjs-exports": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fix-dts-default-cjs-exports/-/fix-dts-default-cjs-exports-1.0.1.tgz", + "integrity": "sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==", + "dev": true, + "dependencies": { + "magic-string": "^0.30.17", + "mlly": "^1.7.4", + "rollup": "^4.34.8" + } + }, + "node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dev": true, + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dev": true, + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/grad-school": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/grad-school/-/grad-school-0.0.5.tgz", + "integrity": "sha512-rXunEHF9M9EkMydTBux7+IryYXEZinRk6g8OBOGDBzo/qWJjhTxy86i5q7lQYpCLHN8Sqv1XX3OIOc7ka2gtvQ==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true + }, + "node_modules/load-tsconfig": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/load-tsconfig/-/load-tsconfig-0.2.5.tgz", + "integrity": "sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/magicast": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.1.tgz", + "integrity": "sha512-xrHS24IxaLrvuo613F719wvOIv9xPHFWQHuvGUBmPnCA/3MQxKI3b+r7n1jAoDHmsbC5bRhTZYR77invLAxVnw==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.28.5", + "@babel/types": "^7.28.5", + "source-map-js": "^1.2.1" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mlly": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz", + "integrity": "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==", + "dev": true, + "dependencies": { + "acorn": "^8.15.0", + "pathe": "^2.0.3", + "pkg-types": "^1.3.1", + "ufo": "^1.6.1" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, + "node_modules/normalize-strings": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/normalize-strings/-/normalize-strings-1.1.1.tgz", + "integrity": "sha512-fARPRdTwmrQDLYhmeh7j/eZwrCP6WzxD6uKOdK/hT/uKACAE9AG2Bc2dgqOZLkfmmctHpfcJ9w3AQnfLgg3GYg==" + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/obug": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", + "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ] + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", + "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-types": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", + "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", + "dev": true, + "dependencies": { + "confbox": "^0.1.8", + "mlly": "^1.7.4", + "pathe": "^2.0.1" + } + }, + "node_modules/pluralize": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", + "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-load-config": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", + "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "lilconfig": "^3.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "jiti": ">=1.21.0", + "postcss": ">=8.0.9", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + }, + "postcss": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rollup": { + "version": "4.55.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.55.1.tgz", + "integrity": "sha512-wDv/Ht1BNHB4upNbK74s9usvl7hObDnvVzknxqY/E/O3X6rW1U1rV1aENEfJ54eFZDTNo7zv1f5N4edCluH7+A==", + "dev": true, + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.55.1", + "@rollup/rollup-android-arm64": "4.55.1", + "@rollup/rollup-darwin-arm64": "4.55.1", + "@rollup/rollup-darwin-x64": "4.55.1", + "@rollup/rollup-freebsd-arm64": "4.55.1", + "@rollup/rollup-freebsd-x64": "4.55.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.55.1", + "@rollup/rollup-linux-arm-musleabihf": "4.55.1", + "@rollup/rollup-linux-arm64-gnu": "4.55.1", + "@rollup/rollup-linux-arm64-musl": "4.55.1", + "@rollup/rollup-linux-loong64-gnu": "4.55.1", + "@rollup/rollup-linux-loong64-musl": "4.55.1", + "@rollup/rollup-linux-ppc64-gnu": "4.55.1", + "@rollup/rollup-linux-ppc64-musl": "4.55.1", + "@rollup/rollup-linux-riscv64-gnu": "4.55.1", + "@rollup/rollup-linux-riscv64-musl": "4.55.1", + "@rollup/rollup-linux-s390x-gnu": "4.55.1", + "@rollup/rollup-linux-x64-gnu": "4.55.1", + "@rollup/rollup-linux-x64-musl": "4.55.1", + "@rollup/rollup-openbsd-x64": "4.55.1", + "@rollup/rollup-openharmony-arm64": "4.55.1", + "@rollup/rollup-win32-arm64-msvc": "4.55.1", + "@rollup/rollup-win32-ia32-msvc": "4.55.1", + "@rollup/rollup-win32-x64-gnu": "4.55.1", + "@rollup/rollup-win32-x64-msvc": "4.55.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", + "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", + "dev": true, + "engines": { + "node": ">= 12" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/sucrase": { + "version": "3.35.1", + "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", + "integrity": "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.2", + "commander": "^4.0.0", + "lines-and-columns": "^1.1.6", + "mz": "^2.7.0", + "pirates": "^4.0.1", + "tinyglobby": "^0.2.11", + "ts-interface-checker": "^0.1.9" + }, + "bin": { + "sucrase": "bin/sucrase", + "sucrase-node": "bin/sucrase-node" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/suffix-thumb": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/suffix-thumb/-/suffix-thumb-5.0.2.tgz", + "integrity": "sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/syllable": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/syllable/-/syllable-5.0.1.tgz", + "integrity": "sha512-HWtNCp6v7J8H0lrT8j1HHjfOLltRoDcC7QRFVu25p4BE52JqetXG65nqC7CsatT8WQRfY4Qvh93BWJIUxbmXFg==", + "dependencies": { + "@types/pluralize": "^0.0.29", + "normalize-strings": "^1.1.0", + "pluralize": "^8.0.0" + }, + "bin": { + "syllable": "cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true + }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "dev": true, + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/tinyrainbow": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.0.3.tgz", + "integrity": "sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==", + "dev": true, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "dev": true, + "bin": { + "tree-kill": "cli.js" + } + }, + "node_modules/ts-api-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", + "integrity": "sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==", + "dev": true, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "typescript": ">=4.2.0" + } + }, + "node_modules/ts-interface-checker": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", + "dev": true + }, + "node_modules/tsup": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/tsup/-/tsup-8.5.1.tgz", + "integrity": "sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==", + "dev": true, + "dependencies": { + "bundle-require": "^5.1.0", + "cac": "^6.7.14", + "chokidar": "^4.0.3", + "consola": "^3.4.0", + "debug": "^4.4.0", + "esbuild": "^0.27.0", + "fix-dts-default-cjs-exports": "^1.0.0", + "joycon": "^3.1.1", + "picocolors": "^1.1.1", + "postcss-load-config": "^6.0.1", + "resolve-from": "^5.0.0", + "rollup": "^4.34.8", + "source-map": "^0.7.6", + "sucrase": "^3.35.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.11", + "tree-kill": "^1.2.2" + }, + "bin": { + "tsup": "dist/cli-default.js", + "tsup-node": "dist/cli-node.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@microsoft/api-extractor": "^7.36.0", + "@swc/core": "^1", + "postcss": "^8.4.12", + "typescript": ">=4.5.0" + }, + "peerDependenciesMeta": { + "@microsoft/api-extractor": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "postcss": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, + "node_modules/tsup/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/ufo": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.2.tgz", + "integrity": "sha512-heMioaxBcG9+Znsda5Q8sQbWnLJSl98AFDXTO80wELWEzX3hordXsTdxrIfMQoO9IY1MEnoGoPjpoKpMj+Yx0Q==", + "dev": true + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.1.tgz", + "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", + "dev": true, + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/vitest": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.0.17.tgz", + "integrity": "sha512-FQMeF0DJdWY0iOnbv466n/0BudNdKj1l5jYgl5JVTwjSsZSlqyXFt/9+1sEyhR6CLowbZpV7O1sCHrzBhucKKg==", + "dev": true, + "dependencies": { + "@vitest/expect": "4.0.17", + "@vitest/mocker": "4.0.17", + "@vitest/pretty-format": "4.0.17", + "@vitest/runner": "4.0.17", + "@vitest/snapshot": "4.0.17", + "@vitest/spy": "4.0.17", + "@vitest/utils": "4.0.17", + "es-module-lexer": "^1.7.0", + "expect-type": "^1.2.2", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^3.10.0", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.0.3", + "vite": "^6.0.0 || ^7.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.0.17", + "@vitest/browser-preview": "4.0.17", + "@vitest/browser-webdriverio": "4.0.17", + "@vitest/ui": "4.0.17", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/vitest/node_modules/tinyexec": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", + "integrity": "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json new file mode 100644 index 0000000..f3e4bf6 --- /dev/null +++ b/sdks/typescript/package.json @@ -0,0 +1,75 @@ +{ + "name": "@learning-commons/evaluators", + "version": "0.1.0", + "description": "TypeScript SDK for Learning Commons educational evaluators", + "type": "module", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs" + } + }, + "sideEffects": false, + "files": [ + "dist", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "tsup", + "dev": "tsup --watch", + "test": "vitest run", + "test:unit": "vitest run tests/unit", + "test:integration": "RUN_INTEGRATION_TESTS=true vitest run tests/integration", + "test:integration:dist": "npm run build && RUN_INTEGRATION_TESTS=true vitest run --config vitest.ci.config.ts", + "test:all": "RUN_INTEGRATION_TESTS=true vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage", + "test:ci": "npm run test:unit && npm run test:integration:dist", + "typecheck": "tsc --noEmit", + "lint": "eslint src --ext .ts", + "prepublishOnly": "npm run build" + }, + "keywords": [ + "education", + "evaluators" + ], + "author": "Learning Commons", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/learning-commons-org/evaluators.git", + "directory": "sdks/typescript" + }, + "bugs": { + "url": "https://github.com/learning-commons-org/evaluators/issues" + }, + "homepage": "https://github.com/learning-commons-org/evaluators#readme", + "peerDependencies": { + "ai": ">=4.0.0" + }, + "dependencies": { + "compromise": "^14.13.0", + "syllable": "^5.0.1", + "zod": "^3.22.4" + }, + "devDependencies": { + "@ai-sdk/anthropic": "^3.0.12", + "@ai-sdk/google": "^3.0.7", + "@ai-sdk/openai": "^3.0.9", + "@types/node": "^20.11.5", + "@typescript-eslint/eslint-plugin": "^6.19.0", + "@typescript-eslint/parser": "^6.19.0", + "@vitest/coverage-v8": "^4.0.17", + "ai": "^6.0.30", + "eslint": "^8.56.0", + "tsup": "^8.0.1", + "typescript": "^5.3.3", + "vitest": "^4.0.17" + }, + "engines": { + "node": ">=20.19.0" + } +} diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts new file mode 100644 index 0000000..cb0ff5c --- /dev/null +++ b/sdks/typescript/src/index.ts @@ -0,0 +1 @@ +export {}; diff --git a/sdks/typescript/src/types/txt.d.ts b/sdks/typescript/src/types/txt.d.ts new file mode 100644 index 0000000..85b042e --- /dev/null +++ b/sdks/typescript/src/types/txt.d.ts @@ -0,0 +1,4 @@ +declare module '*.txt' { + const content: string; + export default content; +} diff --git a/sdks/typescript/tsconfig.json b/sdks/typescript/tsconfig.json new file mode 100644 index 0000000..a5c68cc --- /dev/null +++ b/sdks/typescript/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022"], + "types": ["node"], + "moduleResolution": "bundler", + "resolveJsonModule": true, + "allowJs": false, + "checkJs": false, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "allowSyntheticDefaultImports": true, + "isolatedModules": true + }, + "include": ["src/**/*", "tests/**/*", "*.config.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/sdks/typescript/tsup.config.ts b/sdks/typescript/tsup.config.ts new file mode 100644 index 0000000..1a81469 --- /dev/null +++ b/sdks/typescript/tsup.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/index.ts'], + format: ['esm', 'cjs'], + dts: true, + splitting: false, + sourcemap: true, + clean: true, + treeshake: true, + minify: false, + external: ['ai', '@ai-sdk/openai', '@ai-sdk/anthropic', '@ai-sdk/google'], + loader: { + '.txt': 'text', // Inline prompt .txt files as strings at build time + }, +}); diff --git a/sdks/typescript/vitest.ci.config.ts b/sdks/typescript/vitest.ci.config.ts new file mode 100644 index 0000000..df4fd6d --- /dev/null +++ b/sdks/typescript/vitest.ci.config.ts @@ -0,0 +1,34 @@ +import { defineConfig } from 'vitest/config'; +import { resolve } from 'path'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +/** + * Vitest configuration for testing the built distribution + * + * This config is used in CI to test the actual package that will be published. + * It remaps imports from src/ to dist/index.js since the build bundles everything. + * + * Usage: vitest run --config vitest.ci.config.ts + */ +export default defineConfig({ + test: { + globals: true, + environment: 'node', + include: ['tests/integration/**/*.test.ts'], + testTimeout: 120000, // 2 minutes for integration tests with LLM calls + }, + resolve: { + alias: [ + // Remap all src/evaluators imports to the bundled dist/index.js + // Example: ../../src/evaluators/vocabulary.js → ../../dist/index.js + { + find: /^.*\/src\/evaluators\/.*\.js$/, + replacement: resolve(__dirname, './dist/index.js'), + }, + ], + }, +}); diff --git a/sdks/typescript/vitest.config.ts b/sdks/typescript/vitest.config.ts new file mode 100644 index 0000000..9eb9a49 --- /dev/null +++ b/sdks/typescript/vitest.config.ts @@ -0,0 +1,39 @@ +import { defineConfig } from 'vitest/config'; +import { readFileSync } from 'fs'; +import { resolve, dirname } from 'path'; +import type { Plugin } from 'vite'; + +function txtPlugin(): Plugin { + return { + name: 'txt-loader', + enforce: 'pre', + resolveId(source, importer) { + if (!source.endsWith('.txt') || !importer) return; + return resolve(dirname(importer), source); + }, + load(id) { + if (!id.endsWith('.txt')) return; + return `export default ${JSON.stringify(readFileSync(id, 'utf-8'))};`; + }, + }; +} + +export default defineConfig({ + plugins: [txtPlugin()], + test: { + globals: true, + environment: 'node', + passWithNoTests: true, + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.config.ts', + '**/*.d.ts', + ], + }, + }, +}); From 35f473c7c2649179784ef716fb924f13896537ac Mon Sep 17 00:00:00 2001 From: Adnan Hussain <61515575+adnanrhussain@users.noreply.github.com> Date: Mon, 2 Mar 2026 22:11:59 -0800 Subject: [PATCH 3/9] feat: Implement core evaluator files and Vocab implementation (#12) * feat: Implement core evaluator files and Vocab implementation --- evals/prompts/vocabulary/grades-3-4-user.txt | 2 - .../prompts/vocabulary/other-grades-user.txt | 2 - sdks/typescript/README.md | 178 +++++++++ sdks/typescript/docs/telemetry.md | 124 ++++++ sdks/typescript/package.json | 10 +- sdks/typescript/src/errors.ts | 263 +++++++++++++ sdks/typescript/src/evaluators/base.ts | 256 +++++++++++++ sdks/typescript/src/evaluators/index.ts | 7 + sdks/typescript/src/evaluators/vocabulary.ts | 353 ++++++++++++++++++ sdks/typescript/src/features/index.ts | 5 + sdks/typescript/src/features/readability.ts | 49 +++ sdks/typescript/src/index.ts | 56 ++- sdks/typescript/src/logger.ts | 159 ++++++++ .../vocabulary/background-knowledge.ts | 10 + .../src/prompts/vocabulary/index.ts | 3 + .../src/prompts/vocabulary/system.ts | 17 + .../typescript/src/prompts/vocabulary/user.ts | 28 ++ .../src/providers/ai-sdk-provider.ts | 144 +++++++ sdks/typescript/src/providers/base.ts | 73 ++++ sdks/typescript/src/providers/index.ts | 10 + sdks/typescript/src/schemas/index.ts | 10 + sdks/typescript/src/schemas/outputs.ts | 75 ++++ sdks/typescript/src/schemas/vocabulary.ts | 39 ++ sdks/typescript/src/telemetry/client.ts | 64 ++++ sdks/typescript/src/telemetry/index.ts | 10 + sdks/typescript/src/telemetry/types.ts | 92 +++++ sdks/typescript/src/telemetry/utils.ts | 93 +++++ sdks/typescript/tests/README.md | 221 +++++++++++ .../vocabulary.integration.test.ts | 141 +++++++ .../tests/unit/evaluators/validation.test.ts | 142 +++++++ .../tests/unit/evaluators/vocabulary.test.ts | 250 +++++++++++++ .../tests/unit/features/readability.test.ts | 27 ++ .../tests/unit/telemetry/utils.test.ts | 121 ++++++ sdks/typescript/tests/utils/index.ts | 18 + sdks/typescript/tests/utils/test-helpers.ts | 254 +++++++++++++ sdks/typescript/vitest.config.ts | 6 +- 36 files changed, 3304 insertions(+), 8 deletions(-) create mode 100644 sdks/typescript/docs/telemetry.md create mode 100644 sdks/typescript/src/errors.ts create mode 100644 sdks/typescript/src/evaluators/base.ts create mode 100644 sdks/typescript/src/evaluators/index.ts create mode 100644 sdks/typescript/src/evaluators/vocabulary.ts create mode 100644 sdks/typescript/src/features/index.ts create mode 100644 sdks/typescript/src/features/readability.ts create mode 100644 sdks/typescript/src/logger.ts create mode 100644 sdks/typescript/src/prompts/vocabulary/background-knowledge.ts create mode 100644 sdks/typescript/src/prompts/vocabulary/index.ts create mode 100644 sdks/typescript/src/prompts/vocabulary/system.ts create mode 100644 sdks/typescript/src/prompts/vocabulary/user.ts create mode 100644 sdks/typescript/src/providers/ai-sdk-provider.ts create mode 100644 sdks/typescript/src/providers/base.ts create mode 100644 sdks/typescript/src/providers/index.ts create mode 100644 sdks/typescript/src/schemas/index.ts create mode 100644 sdks/typescript/src/schemas/outputs.ts create mode 100644 sdks/typescript/src/schemas/vocabulary.ts create mode 100644 sdks/typescript/src/telemetry/client.ts create mode 100644 sdks/typescript/src/telemetry/index.ts create mode 100644 sdks/typescript/src/telemetry/types.ts create mode 100644 sdks/typescript/src/telemetry/utils.ts create mode 100644 sdks/typescript/tests/README.md create mode 100644 sdks/typescript/tests/integration/vocabulary.integration.test.ts create mode 100644 sdks/typescript/tests/unit/evaluators/validation.test.ts create mode 100644 sdks/typescript/tests/unit/evaluators/vocabulary.test.ts create mode 100644 sdks/typescript/tests/unit/features/readability.test.ts create mode 100644 sdks/typescript/tests/unit/telemetry/utils.test.ts create mode 100644 sdks/typescript/tests/utils/index.ts create mode 100644 sdks/typescript/tests/utils/test-helpers.ts diff --git a/evals/prompts/vocabulary/grades-3-4-user.txt b/evals/prompts/vocabulary/grades-3-4-user.txt index 1759511..7da9831 100644 --- a/evals/prompts/vocabulary/grades-3-4-user.txt +++ b/evals/prompts/vocabulary/grades-3-4-user.txt @@ -10,5 +10,3 @@ Below is the text you need to evaluate. Let's think step by step in order to pre - Text to evaluate: [BEGIN TEXT] {text} [END TEXT] - -{format_instructions} diff --git a/evals/prompts/vocabulary/other-grades-user.txt b/evals/prompts/vocabulary/other-grades-user.txt index 0d4b534..95cc176 100644 --- a/evals/prompts/vocabulary/other-grades-user.txt +++ b/evals/prompts/vocabulary/other-grades-user.txt @@ -135,5 +135,3 @@ As you read the text, you can assume the student has the following background kn [END TEXT] In your response, when specifying the level of complexity, be sure to use only a single integer (e.g. 2) and don't include any other text (e.g. don't say "level 2"). - -{format_instructions} diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md index 3ea9d1c..f4209df 100644 --- a/sdks/typescript/README.md +++ b/sdks/typescript/README.md @@ -1 +1,179 @@ # @learning-commons/evaluators + +TypeScript SDK for Learning Commons educational text complexity evaluators. + +## Installation + +```bash +npm install @learning-commons/evaluators ai +``` + +The SDK uses the [Vercel AI SDK](https://sdk.vercel.ai) (`ai`) as its LLM interface. You also need to install the provider adapter(s) for the LLM(s) you use: + +```bash +npm install @ai-sdk/openai # for OpenAI +npm install @ai-sdk/google # for Google Gemini +npm install @ai-sdk/anthropic # for Anthropic +``` + +## Quick Start + +```typescript +import { VocabularyEvaluator } from '@learning-commons/evaluators'; + +const evaluator = new VocabularyEvaluator({ + googleApiKey: process.env.GOOGLE_API_KEY, + openaiApiKey: process.env.OPENAI_API_KEY +}); + +const result = await evaluator.evaluate("Your text here", "5"); +console.log(result.score); // "moderately complex" +``` + +--- + +## Evaluators + +### 1. Vocabulary Evaluator + +Evaluates vocabulary complexity using the Qual Text Complexity rubric (SAP). + +**Supported Grades:** 3-12 + +**Uses:** Google Gemini 2.5 Pro + OpenAI GPT-4o + +**Constructor:** +```typescript +const evaluator = new VocabularyEvaluator({ + googleApiKey: string; // Required - Google API key + openaiApiKey: string; // Required - OpenAI API key + maxRetries?: number; // Optional - Max retry attempts (default: 2) + telemetry?: boolean | TelemetryOptions; // Optional (default: true) + logger?: Logger; // Optional - Custom logger + logLevel?: LogLevel; // Optional - SILENT | ERROR | WARN | INFO | DEBUG (default: WARN) +}); +``` + +**API:** +```typescript +await evaluator.evaluate(text: string, grade: string) +``` + +**Returns:** +```typescript +{ + score: 'slightly complex' | 'moderately complex' | 'very complex' | 'exceedingly complex'; + reasoning: string; + metadata: { + promptVersion: string; + model: string; + timestamp: Date; + processingTimeMs: number; + }; + _internal: VocabularyComplexity; // Detailed analysis +} +``` + +## Error Handling + +The SDK provides specific error types to help you handle different scenarios: + +```typescript +import { + ConfigurationError, + ValidationError, + APIError, + AuthenticationError, + RateLimitError, + NetworkError, + TimeoutError, +} from '@learning-commons/evaluators'; + +try { + const evaluator = new VocabularyEvaluator({ googleApiKey, openaiApiKey }); + const result = await evaluator.evaluate(text, grade); +} catch (error) { + if (error instanceof ConfigurationError) { + // Missing or invalid API keys — fix your config + console.error('Configuration error:', error.message); + } else if (error instanceof ValidationError) { + // Invalid input (text too short, invalid grade, etc.) + console.error('Invalid input:', error.message); + } else if (error instanceof AuthenticationError) { + // Invalid API keys + console.error('Check your API keys:', error.message); + } else if (error instanceof RateLimitError) { + // Rate limit exceeded - wait and retry + console.error('Rate limited. Retry after:', error.retryAfter); + } else if (error instanceof NetworkError) { + // Network connectivity issues + console.error('Network error:', error.message); + } else if (error instanceof APIError) { + // Other API errors + console.error('API error:', error.message, 'Status:', error.statusCode); + } +} +``` + +--- + +## Logging + +Control logging verbosity with `logLevel`: + +```typescript +import { VocabularyEvaluator, LogLevel } from '@learning-commons/evaluators'; + +const evaluator = new VocabularyEvaluator({ + googleApiKey: '...', + openaiApiKey: '...', + logLevel: LogLevel.INFO, // SILENT | ERROR | WARN | INFO | DEBUG +}); +``` + +Or provide a custom logger: + +```typescript +import type { Logger } from '@learning-commons/evaluators'; + +const customLogger: Logger = { + debug: (msg, ctx) => myLogger.debug(msg, ctx), + info: (msg, ctx) => myLogger.info(msg, ctx), + warn: (msg, ctx) => myLogger.warn(msg, ctx), + error: (msg, ctx) => myLogger.error(msg, ctx), +}; + +const evaluator = new VocabularyEvaluator({ + googleApiKey: '...', + openaiApiKey: '...', + logger: customLogger, +}); +``` + +--- + +## Telemetry & Privacy + +See [docs/telemetry.md](./docs/telemetry.md) for telemetry configuration and privacy information. + +--- + +## Configuration Options + +All evaluators support these common options: + +```typescript +interface BaseEvaluatorConfig { + maxRetries?: number; // Max API retry attempts (default: 2) + telemetry?: boolean | TelemetryOptions; // Telemetry config (default: true) + logger?: Logger; // Custom logger (optional) + logLevel?: LogLevel; // Console log level (default: WARN) + partnerKey?: string; // Learning Commons partner key for authenticated telemetry (optional) +} +``` + +--- + +## License + +MIT diff --git a/sdks/typescript/docs/telemetry.md b/sdks/typescript/docs/telemetry.md new file mode 100644 index 0000000..991479f --- /dev/null +++ b/sdks/typescript/docs/telemetry.md @@ -0,0 +1,124 @@ +# Telemetry + +## Why We Collect Telemetry + +We use telemetry data to improve evaluator quality, identify edge cases, and optimize performance. This helps us build better tools for our developer partners. + +Telemetry is **anonymous by default**. If you'd like to partner with us to improve your specific use case, you can optionally provide an API key (see Configuration section below). This allows us to connect with you and collaborate more deeply. + +## What We Collect + +**By default, telemetry is enabled** and sends: +- Performance metrics (latency, token usage) +- Metadata (evaluator type, grade, SDK version) + +**Input text is NOT collected by default.** You can opt in via `recordInputs: true` — see [Enable Input Text Collection](#enable-input-text-collection) below. + +We **never** collect your API keys (only an anonymous identifier). + +If you prefer not to send any telemetry, you can disable it entirely — see [Disable Telemetry Completely](#disable-telemetry-completely) below. + +## Example Telemetry Event + +```json +{ + "timestamp": "2026-02-05T19:30:00.000Z", + "sdk_version": "0.1.0", + "evaluator_type": "vocabulary", + "grade": "5", + "status": "success", + "latency_ms": 3500, + "text_length_chars": 456, + "provider": "google:gemini-2.5-pro+openai:gpt-4o", + "token_usage": { + "input_tokens": 650, + "output_tokens": 350 + }, + "metadata": { + "stage_details": [ + { + "stage": "background_knowledge", + "provider": "openai:gpt-4o-2024-11-20", + "latency_ms": 1200, + "token_usage": { + "input_tokens": 250, + "output_tokens": 150 + } + }, + { + "stage": "complexity_evaluation", + "provider": "google:gemini-2.5-pro", + "latency_ms": 2300, + "token_usage": { + "input_tokens": 400, + "output_tokens": 200 + } + } + ] + } +} +``` + +## Field Reference + +| Field | Description | +|-------|-------------| +| `timestamp` | ISO 8601 timestamp when evaluation started | +| `sdk_version` | Version of the SDK (e.g., "0.1.0") | +| `evaluator_type` | Which evaluator ran (e.g., "vocabulary", "sentence-structure") | +| `grade` | Grade level evaluated (e.g., "5", "K") | +| `status` | Evaluation outcome: "success" or "error" | +| `error_code` | Error type if status is "error" (e.g., "Error", "TypeError") | +| `latency_ms` | Total evaluation time in milliseconds | +| `text_length_chars` | Length of input text in characters | +| `provider` | LLM provider(s) used (e.g., "openai:gpt-4o", "google:gemini-2.5-pro+openai:gpt-4o") | +| `token_usage` | Total tokens consumed (input, output, total) | +| `input_text` | The text being evaluated (only included if `recordInputs: true`) | +| `metadata.stage_details` | Per-stage breakdown for multi-stage evaluators (optional) | + +## Configuration + +### Default (Anonymous) + +```typescript +const evaluator = new VocabularyEvaluator({ + googleApiKey: process.env.GOOGLE_API_KEY!, + openaiApiKey: process.env.OPENAI_API_KEY!, + // telemetry: true (default - anonymous) +}); +``` + +### Partner with Us (Authenticated) + +To help us support your specific use case, provide an API key: + +```typescript +const evaluator = new VocabularyEvaluator({ + googleApiKey: process.env.GOOGLE_API_KEY!, + openaiApiKey: process.env.OPENAI_API_KEY!, + partnerKey: process.env.LEARNING_COMMONS_PARTNER_KEY!, // Contact us for a key +}); +``` + +### Disable Telemetry Completely + +```typescript +const evaluator = new VocabularyEvaluator({ + googleApiKey: process.env.GOOGLE_API_KEY!, + openaiApiKey: process.env.OPENAI_API_KEY!, + telemetry: false, // No data sent +}); +``` + +### Enable Input Text Collection + +```typescript +const evaluator = new VocabularyEvaluator({ + googleApiKey: process.env.GOOGLE_API_KEY!, + openaiApiKey: process.env.OPENAI_API_KEY!, + telemetry: { + enabled: true, + recordInputs: true, // Also send input text with telemetry + }, +}); +``` diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json index f3e4bf6..b269046 100644 --- a/sdks/typescript/package.json +++ b/sdks/typescript/package.json @@ -48,7 +48,15 @@ }, "homepage": "https://github.com/learning-commons-org/evaluators#readme", "peerDependencies": { - "ai": ">=4.0.0" + "ai": ">=6.0.0", + "@ai-sdk/openai": ">=3.0.0", + "@ai-sdk/google": ">=3.0.0", + "@ai-sdk/anthropic": ">=3.0.0" + }, + "peerDependenciesMeta": { + "@ai-sdk/openai": { "optional": true }, + "@ai-sdk/google": { "optional": true }, + "@ai-sdk/anthropic": { "optional": true } }, "dependencies": { "compromise": "^14.13.0", diff --git a/sdks/typescript/src/errors.ts b/sdks/typescript/src/errors.ts new file mode 100644 index 0000000..f31828a --- /dev/null +++ b/sdks/typescript/src/errors.ts @@ -0,0 +1,263 @@ +/** + * Custom error types for the Evaluators SDK + * + * This module provides a hierarchy of error types to help users + * distinguish between different error scenarios and implement + * appropriate error handling strategies. + */ + +/** + * Base error class for all evaluator errors + */ +export class EvaluatorError extends Error { + constructor( + message: string, + public readonly code?: string + ) { + super(message); + this.name = 'EvaluatorError'; + // Maintains proper stack trace for where error was thrown (only available on V8) + if (Error.captureStackTrace) { + Error.captureStackTrace(this, this.constructor); + } + } +} + +/** + * Configuration error - thrown when the evaluator is misconfigured + * These are developer errors (e.g. missing API keys) that should NOT be retried + * + * @example + * ```typescript + * try { + * const evaluator = new VocabularyEvaluator({ googleApiKey: '' }); + * } catch (error) { + * if (error instanceof ConfigurationError) { + * console.error('Check your evaluator config:', error.message); + * } + * } + * ``` + */ +export class ConfigurationError extends EvaluatorError { + constructor(message: string) { + super(message, 'CONFIGURATION_ERROR'); + this.name = 'ConfigurationError'; + } +} + +/** + * Validation error - thrown when input validation fails + * These are client-side errors that should NOT be retried + * + * @example + * ```typescript + * try { + * await evaluator.evaluate('', '5'); + * } catch (error) { + * if (error instanceof ValidationError) { + * // Show user-friendly error message + * console.error('Invalid input:', error.message); + * } + * } + * ``` + */ +export class ValidationError extends EvaluatorError { + constructor(message: string) { + super(message, 'VALIDATION_ERROR'); + this.name = 'ValidationError'; + } +} + +/** + * Base API error - thrown when LLM API calls fail + * Contains additional context about the API error + */ +export class APIError extends EvaluatorError { + constructor( + message: string, + public readonly statusCode?: number, + public readonly retryable: boolean = false, + code?: string + ) { + super(message, code); + this.name = 'APIError'; + } +} + +/** + * Authentication error - thrown when API keys are invalid or missing + * HTTP 401 or 403 responses + * Should NOT be retried + * + * @example + * ```typescript + * try { + * await evaluator.evaluate(text, grade); + * } catch (error) { + * if (error instanceof AuthenticationError) { + * // Prompt user to check API keys + * console.error('Invalid API keys. Please check your credentials.'); + * } + * } + * ``` + */ +export class AuthenticationError extends APIError { + constructor(message: string, statusCode?: number) { + super(message, statusCode, false, 'AUTHENTICATION_ERROR'); + this.name = 'AuthenticationError'; + } +} + +/** + * Rate limit error - thrown when API rate limits are exceeded + * HTTP 429 responses + * Should be retried with exponential backoff + * + * @example + * ```typescript + * try { + * await evaluator.evaluate(text, grade); + * } catch (error) { + * if (error instanceof RateLimitError) { + * // Wait and retry + * await sleep(error.retryAfter || 5000); + * // retry... + * } + * } + * ``` + */ +export class RateLimitError extends APIError { + constructor( + message: string, + public readonly retryAfter?: number // milliseconds + ) { + super(message, 429, true, 'RATE_LIMIT_ERROR'); + this.name = 'RateLimitError'; + } +} + +/** + * Network error - thrown when network requests fail + * Connection timeouts, DNS failures, etc. + * May be retryable depending on the scenario + * + * @example + * ```typescript + * try { + * await evaluator.evaluate(text, grade); + * } catch (error) { + * if (error instanceof NetworkError) { + * // Check network connection and retry + * console.error('Network error:', error.message); + * } + * } + * ``` + */ +export class NetworkError extends APIError { + constructor(message: string, retryable: boolean = true) { + super(message, undefined, retryable, 'NETWORK_ERROR'); + this.name = 'NetworkError'; + } +} + +/** + * Timeout error - thrown when requests exceed timeout limits + * Should be retried with caution + * + * @example + * ```typescript + * try { + * await evaluator.evaluate(text, grade); + * } catch (error) { + * if (error instanceof TimeoutError) { + * // Retry with longer timeout or smaller text + * console.error('Request timed out'); + * } + * } + * ``` + */ +export class TimeoutError extends APIError { + constructor(message: string = 'Request timed out') { + super(message, 408, true, 'TIMEOUT_ERROR'); + this.name = 'TimeoutError'; + } +} + +/** + * Parse structured output from LLM provider error + */ +function parseProviderError(error: unknown): { message: string; statusCode?: number; code?: string } { + // Handle Error objects + if (error instanceof Error) { + const message = error.message; + + // Try to extract status code from error message + // Common patterns: "429", "401", "Error 429:", "Status: 429" + const statusMatch = message.match(/\b(4\d{2}|5\d{2})\b/); + const statusCode = statusMatch ? parseInt(statusMatch[1]) : undefined; + + return { + message, + statusCode, + code: error.name !== 'Error' ? error.name : undefined, + }; + } + + // Handle unknown error types + return { + message: String(error), + }; +} + +/** + * Wrap a provider error into the appropriate error type + * + * @internal + */ +export function wrapProviderError(error: unknown, defaultMessage: string = 'API request failed'): APIError { + const { message, statusCode, code } = parseProviderError(error); + + // Detect authentication errors (401, 403) + if (statusCode === 401 || statusCode === 403) { + return new AuthenticationError( + message.includes('API key') ? message : 'Invalid API key', + statusCode + ); + } + + // Detect rate limit errors (429) + if (statusCode === 429) { + // Try to extract retry-after if present + const retryAfterMatch = message.match(/retry[- ]after[:\s]+(\d+)/i); + const retryAfter = retryAfterMatch ? parseInt(retryAfterMatch[1]) * 1000 : undefined; + + return new RateLimitError( + message.includes('rate limit') ? message : 'Rate limit exceeded', + retryAfter + ); + } + + // Detect network errors + if ( + message.includes('ECONNREFUSED') || + message.includes('ENOTFOUND') || + message.includes('ETIMEDOUT') || + message.includes('network') || + message.includes('Network') + ) { + return new NetworkError(message); + } + + // Detect timeout errors + if (message.includes('timeout') || message.includes('timed out')) { + return new TimeoutError(message); + } + + // Generic API error for everything else + return new APIError( + message || defaultMessage, + statusCode, + statusCode ? statusCode >= 500 : false, // 5xx errors are retryable + code + ); +} diff --git a/sdks/typescript/src/evaluators/base.ts b/sdks/typescript/src/evaluators/base.ts new file mode 100644 index 0000000..bef2ec4 --- /dev/null +++ b/sdks/typescript/src/evaluators/base.ts @@ -0,0 +1,256 @@ +import { + TelemetryClient, + generateClientId, + getSDKVersion, + type TelemetryMetadata, + type TokenUsage, +} from '../telemetry/index.js'; +import { ValidationError } from '../errors.js'; +import { createLogger, LogLevel, type Logger } from '../logger.js'; + +/** + * Validation constants for input text + */ +export const VALIDATION_LIMITS = { + /** Minimum text length in characters */ + MIN_TEXT_LENGTH: 10, + /** Maximum text length in characters (100K chars ≈ 25K tokens) */ + MAX_TEXT_LENGTH: 100_000, +} as const; + +/** + * Granular telemetry configuration options + */ +export interface TelemetryOptions { + /** Enable telemetry (default: true) */ + enabled?: boolean; + + /** Record input text in telemetry (default: false) */ + recordInputs?: boolean; +} + +/** + * Base configuration for all evaluators + */ +export interface BaseEvaluatorConfig { + /** Google API key (for evaluators using Gemini) */ + googleApiKey?: string; + + /** OpenAI API key (for evaluators using GPT) */ + openaiApiKey?: string; + + /** Learning Commons partner key for authenticated telemetry (optional) */ + partnerKey?: string; + + /** + * Maximum number of retries for failed API calls (default: 2) + * Set to 0 to disable retries. + * + * Note: With maxRetries=2, a failed call will be attempted up to 3 times total + * (1 initial attempt + 2 retries) + */ + maxRetries?: number; + + /** + * Telemetry configuration (default: all enabled) + * + * Can be: + * - `true`: Enable with defaults (recordInputs: false) + * - `false`: Disable completely + * - `TelemetryOptions`: Granular control + */ + telemetry?: boolean | TelemetryOptions; + + /** + * Custom logger implementation (optional) + * If not provided, uses console logger with specified logLevel + */ + logger?: Logger; + + /** + * Log level for default console logger (default: WARN) + * Only used if custom logger is not provided + * + * - DEBUG: Very verbose, shows all operations + * - INFO: Normal operations + * - WARN: Warnings only (default) + * - ERROR: Errors only + * - SILENT: No logging + */ + logLevel?: LogLevel; +} + +/** + * Abstract base class for all evaluators + * + * Provides common functionality: + * - Telemetry setup and event sending + * - Text validation + * - Grade validation (with overridable default) + * - Metadata creation + */ +export abstract class BaseEvaluator { + protected telemetryClient?: TelemetryClient; + protected logger: Logger; + protected config: Required> & { + telemetry: Required; + }; + + constructor(config: BaseEvaluatorConfig) { + // Initialize logger + this.logger = createLogger(config.logger, config.logLevel ?? LogLevel.WARN); + // Normalize telemetry config + const telemetryConfig = this.normalizeTelemetryConfig(config.telemetry); + + // Set defaults for common config + this.config = { + maxRetries: config.maxRetries ?? 2, + telemetry: telemetryConfig, + }; + + // Initialize telemetry if enabled + if (this.config.telemetry.enabled) { + this.telemetryClient = new TelemetryClient({ + endpoint: 'https://api.learningcommons.org/v1/telemetry', + partnerKey: config.partnerKey, + clientId: generateClientId(), + enabled: true, + logger: this.logger, + }); + } + } + + /** + * Normalize telemetry config to standard format + */ + private normalizeTelemetryConfig( + telemetry: boolean | TelemetryOptions | undefined + ): Required { + // Handle boolean shortcuts + if (telemetry === false) { + return { + enabled: false, + recordInputs: false, + }; + } + + if (telemetry === true || telemetry === undefined) { + return { + enabled: true, + recordInputs: false, + }; + } + + // Handle granular config object + return { + enabled: telemetry.enabled ?? true, + recordInputs: telemetry.recordInputs ?? false, + }; + } + + /** + * Get the evaluator type identifier (e.g., "vocabulary", "sentence-structure") + * Must be implemented by concrete evaluators + */ + protected abstract getEvaluatorType(): string; + + /** + * Validate text meets requirements + * Default implementation - can be overridden by concrete evaluators + * + * @throws {ValidationError} If text is invalid + */ + protected validateText(text: string): void { + this.logger.debug('Validating text input', { + evaluator: this.getEvaluatorType(), + operation: 'validateText', + textLength: text.length, + }); + + // Check if text is empty or only whitespace + const trimmedText = text.trim(); + if (!trimmedText) { + throw new ValidationError('Text cannot be empty or contain only whitespace'); + } + + // Check minimum length + if (trimmedText.length < VALIDATION_LIMITS.MIN_TEXT_LENGTH) { + throw new ValidationError( + `Text is too short. Minimum length is ${VALIDATION_LIMITS.MIN_TEXT_LENGTH} characters, received ${trimmedText.length} characters` + ); + } + + // Check maximum length + if (trimmedText.length > VALIDATION_LIMITS.MAX_TEXT_LENGTH) { + throw new ValidationError( + `Text is too long. Maximum length is ${VALIDATION_LIMITS.MAX_TEXT_LENGTH.toLocaleString()} characters, received ${trimmedText.length.toLocaleString()} characters` + ); + } + } + + /** + * Validate grade is in supported range + * Default implementation - can be overridden by concrete evaluators + * + * @param grade - Grade level to validate + * @param validGrades - Set of valid grades for this evaluator + * @throws {ValidationError} If grade is invalid + */ + protected validateGrade(grade: string, validGrades: Set): void { + this.logger.debug('Validating grade input', { + evaluator: this.getEvaluatorType(), + operation: 'validateGrade', + grade, + }); + + // Check if grade is in valid set + if (!validGrades.has(grade)) { + const validList = Array.from(validGrades).sort((a, b) => { + // Sort K first, then numerically + if (a === 'K') return -1; + if (b === 'K') return 1; + return parseInt(a) - parseInt(b); + }).join(', '); + + throw new ValidationError( + `Invalid grade "${grade}". Supported grades for this evaluator: ${validList}` + ); + } + } + + /** + * Send telemetry event to analytics service + * Common helper for all evaluators + */ + protected async sendTelemetry(params: { + status: 'success' | 'error'; + latencyMs: number; + textLength: number; + grade?: string; + provider: string; + errorCode?: string; + tokenUsage?: TokenUsage; + metadata?: TelemetryMetadata; + inputText?: string; + }): Promise { + if (!this.telemetryClient) { + return; + } + + await this.telemetryClient.send({ + timestamp: new Date().toISOString(), + sdk_version: getSDKVersion(), + evaluator_type: this.getEvaluatorType(), + grade: params.grade, + status: params.status, + error_code: params.errorCode, + latency_ms: params.latencyMs, + text_length_chars: params.textLength, + provider: params.provider, + token_usage: params.tokenUsage, + metadata: params.metadata, + // Include input text only if recording is enabled + input_text: this.config.telemetry.recordInputs ? params.inputText : undefined, + }); + } +} diff --git a/sdks/typescript/src/evaluators/index.ts b/sdks/typescript/src/evaluators/index.ts new file mode 100644 index 0000000..2f6ff9d --- /dev/null +++ b/sdks/typescript/src/evaluators/index.ts @@ -0,0 +1,7 @@ +export { BaseEvaluator, type BaseEvaluatorConfig, type TelemetryOptions } from './base.js'; + +export { + VocabularyEvaluator, + evaluateVocabulary, + type VocabularyEvaluatorConfig, +} from './vocabulary.js'; diff --git a/sdks/typescript/src/evaluators/vocabulary.ts b/sdks/typescript/src/evaluators/vocabulary.ts new file mode 100644 index 0000000..912e8a2 --- /dev/null +++ b/sdks/typescript/src/evaluators/vocabulary.ts @@ -0,0 +1,353 @@ +import type { LLMProvider } from '../providers/index.js'; +import { createProvider } from '../providers/index.js'; +import { + VocabularyComplexitySchema, + type VocabularyComplexity, + type BackgroundKnowledge, +} from '../schemas/vocabulary.js'; +import { calculateFleschKincaidGrade } from '../features/index.js'; +import { + getBackgroundKnowledgePrompt, + getSystemPrompt, + getUserPrompt, +} from '../prompts/vocabulary/index.js'; +import type { EvaluationResult } from '../schemas/index.js'; +import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; +import type { StageDetail } from '../telemetry/index.js'; +import { ConfigurationError, ValidationError, wrapProviderError } from '../errors.js'; + +/** + * Valid grade levels (3-12) + */ +const VALID_GRADES = new Set(['3', '4', '5', '6', '7', '8', '9', '10', '11', '12']); + +/** + * Configuration for VocabularyEvaluator + */ +export interface VocabularyEvaluatorConfig extends BaseEvaluatorConfig { + /** Google API key for complexity evaluation (uses Gemini 2.5 Pro) */ + googleApiKey: string; + + /** OpenAI API key for background knowledge generation (uses GPT-4o) */ + openaiApiKey: string; +} + +/** + * Vocabulary Evaluator + * + * Evaluates vocabulary complexity of educational texts relative to grade level. + * Uses a 2-stage process: + * 1. Generate background knowledge assumption for the student's grade level + * 2. Evaluate vocabulary complexity using that background knowledge + * + * Based on Qual Text Complexity rubric (SAP) with 4 levels: + * - Slightly complex + * - Moderately complex + * - Very complex + * - Exceedingly complex + * + * @example + * ```typescript + * const evaluator = new VocabularyEvaluator({ + * googleApiKey: process.env.GOOGLE_API_KEY, + * openaiApiKey: process.env.OPENAI_API_KEY + * }); + * + * const result = await evaluator.evaluate(text, "3"); + * console.log(result.score); // "moderately complex" + * console.log(result.reasoning); + * ``` + */ +export class VocabularyEvaluator extends BaseEvaluator { + private grades34ComplexityProvider: LLMProvider; + private otherGradesComplexityProvider: LLMProvider; + private backgroundKnowledgeProvider: LLMProvider; + + constructor(config: VocabularyEvaluatorConfig) { + // Call base constructor for common setup (telemetry, etc.) + super(config); + + // Validate required API keys + if (!config.googleApiKey) { + throw new ConfigurationError('Google API key is required. Pass googleApiKey in config.'); + } + + if (!config.openaiApiKey) { + throw new ConfigurationError('OpenAI API key is required. Pass openaiApiKey in config.'); + } + + // Create Google Gemini provider for complexity evaluation (grades 3-4) + this.grades34ComplexityProvider = createProvider({ + type: 'google', + model: 'gemini-2.5-pro', + apiKey: config.googleApiKey, + maxRetries: this.config.maxRetries, + }); + + // Create OpenAI GPT-4.1 provider for complexity evaluation (grades 5-12) + this.otherGradesComplexityProvider = createProvider({ + type: 'openai', + model: 'gpt-4.1-2025-04-14', + apiKey: config.openaiApiKey, + maxRetries: this.config.maxRetries, + }); + + // Create OpenAI GPT-4o provider for background knowledge generation + this.backgroundKnowledgeProvider = createProvider({ + type: 'openai', + model: 'gpt-4o-2024-11-20', + apiKey: config.openaiApiKey, + maxRetries: this.config.maxRetries, + }); + } + + // Implement abstract methods from BaseEvaluator + protected getEvaluatorType(): string { + return 'vocabulary'; + } + + /** + * Evaluate vocabulary complexity for a given text and grade level + * + * @param text - The text to evaluate + * @param grade - The target grade level (3-12) + * @returns Evaluation result with complexity score and detailed analysis + * @throws {ValidationError} If text is empty, too short/long, or grade is invalid + * @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError) + */ + async evaluate( + text: string, + grade: string + ): Promise> { + this.logger.info('Starting vocabulary evaluation', { + evaluator: 'vocabulary', + operation: 'evaluate', + grade, + textLength: text.length, + }); + + const startTime = Date.now(); + const stageDetails: StageDetail[] = []; + const complexityProviderName = (grade === '3' || grade === '4') + ? 'google:gemini-2.5-pro' + : 'openai:gpt-4.1-2025-04-14'; + + try { + // Validate inputs — inside try so validation errors are telemetered. + // If partners consistently pass invalid grades/text, telemetry will surface documentation gaps. + this.validateText(text); + this.validateGrade(grade, VALID_GRADES); + this.logger.debug('Stage 1: Generating background knowledge', { + evaluator: 'vocabulary', + operation: 'background_knowledge', + }); + // Stage 1: Generate background knowledge assumption + const bgResponse = await this.getBackgroundKnowledgeAssumption(text, grade); + + stageDetails.push({ + stage: 'background_knowledge', + provider: 'openai:gpt-4o-2024-11-20', + latency_ms: bgResponse.latencyMs, + token_usage: { + input_tokens: bgResponse.usage.inputTokens, + output_tokens: bgResponse.usage.outputTokens, + }, + }); + + // Calculate Flesch-Kincaid grade level + const fkLevel = calculateFleschKincaidGrade(text); + + // Stage 2: Evaluate vocabulary complexity + const complexityResponse = await this.evaluateComplexity( + text, + grade, + bgResponse.knowledge.assumption, + fkLevel + ); + + stageDetails.push({ + stage: 'complexity_evaluation', + provider: complexityProviderName, + latency_ms: complexityResponse.latencyMs, + token_usage: { + input_tokens: complexityResponse.usage.inputTokens, + output_tokens: complexityResponse.usage.outputTokens, + }, + }); + + const latencyMs = Date.now() - startTime; + + // Aggregate token usage + const totalTokenUsage = { + input_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.input_tokens || 0), 0), + output_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.output_tokens || 0), 0), + }; + + const result = { + score: complexityResponse.data.complexity_score, + reasoning: complexityResponse.data.reasoning, + metadata: { + promptVersion: '1.2.0', + model: `openai:gpt-4o-2024-11-20 + ${complexityProviderName}`, + timestamp: new Date(), + processingTimeMs: latencyMs, + }, + _internal: complexityResponse.data, + }; + + // Send success telemetry (fire-and-forget) + this.sendTelemetry({ + status: 'success', + latencyMs, + textLength: text.length, + grade, + provider: `openai:gpt-4o-2024-11-20 + ${complexityProviderName}`, + tokenUsage: totalTokenUsage, + metadata: { + stage_details: stageDetails, + }, + inputText: text, + }).catch(() => { + // Ignore telemetry errors + }); + + this.logger.info('Vocabulary evaluation completed successfully', { + evaluator: 'vocabulary', + operation: 'evaluate', + grade, + score: result.score, + processingTimeMs: latencyMs, + }); + + return result; + } catch (error) { + const latencyMs = Date.now() - startTime; + + // Log the error + this.logger.error('Vocabulary evaluation failed', { + evaluator: 'vocabulary', + operation: 'evaluate', + grade, + error: error instanceof Error ? error : undefined, + processingTimeMs: latencyMs, + completedStages: stageDetails.length, + }); + + // Aggregate metrics from completed stages + const totalTokenUsage = stageDetails.length > 0 ? { + input_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.input_tokens || 0), 0), + output_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.output_tokens || 0), 0), + } : undefined; + + // Send failure telemetry (fire-and-forget) + this.sendTelemetry({ + status: 'error', + latencyMs, + textLength: text.length, + grade, + provider: `openai:gpt-4o-2024-11-20 + ${complexityProviderName}`, + tokenUsage: totalTokenUsage, + errorCode: error instanceof Error ? error.name : 'UnknownError', + metadata: stageDetails.length > 0 ? { stage_details: stageDetails } : undefined, + inputText: text, + }).catch(() => { + // Ignore telemetry errors + }); + + // Re-throw validation errors as-is + if (error instanceof ValidationError) { + throw error; + } + + // Wrap provider errors into appropriate error types + throw wrapProviderError(error, 'Vocabulary evaluation failed'); + } + } + + /** + * Stage 1: Generate background knowledge assumption + * + * Estimates what topics the student at the given grade level would be familiar with + * based on Common Core curriculum progression. + */ + private async getBackgroundKnowledgeAssumption( + text: string, + grade: string + ): Promise<{ knowledge: BackgroundKnowledge; usage: { inputTokens: number; outputTokens: number }; latencyMs: number }> { + const prompt = getBackgroundKnowledgePrompt(text, grade); + + const response = await this.backgroundKnowledgeProvider.generateText( + [{ role: 'user', content: prompt }], + 0 // temperature = 0 for consistency + ); + + return { + knowledge: { + assumption: response.text.trim(), + grade, + }, + usage: response.usage, + latencyMs: response.latencyMs, + }; + } + + /** + * Stage 2: Evaluate vocabulary complexity + * + * Uses the Qual Text Complexity rubric (SAP) and background knowledge to evaluate vocabulary complexity. + * Grades 3-4 use Gemini 2.5 Pro; grades 5-12 use GPT-4.1. + */ + private async evaluateComplexity( + text: string, + grade: string, + backgroundKnowledge: string, + fkLevel: number + ): Promise<{ data: VocabularyComplexity; usage: { inputTokens: number; outputTokens: number }; latencyMs: number }> { + const systemPrompt = getSystemPrompt(grade); + const userPrompt = getUserPrompt(text, grade, backgroundKnowledge, fkLevel); + + const provider = (grade === '3' || grade === '4') + ? this.grades34ComplexityProvider + : this.otherGradesComplexityProvider; + + const response = await provider.generateStructured({ + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userPrompt }, + ], + schema: VocabularyComplexitySchema, + temperature: 0, + }); + + return { + data: response.data, + usage: response.usage, + latencyMs: response.latencyMs, + }; + } + +} + +/** + * Functional API for vocabulary evaluation + * + * @example + * ```typescript + * const result = await evaluateVocabulary( + * "The mitochondria is the powerhouse of the cell.", + * "3", + * { + * googleApiKey: process.env.GOOGLE_API_KEY, + * openaiApiKey: process.env.OPENAI_API_KEY + * } + * ); + * ``` + */ +export async function evaluateVocabulary( + text: string, + grade: string, + config: VocabularyEvaluatorConfig +): Promise> { + const evaluator = new VocabularyEvaluator(config); + return evaluator.evaluate(text, grade); +} diff --git a/sdks/typescript/src/features/index.ts b/sdks/typescript/src/features/index.ts new file mode 100644 index 0000000..354830e --- /dev/null +++ b/sdks/typescript/src/features/index.ts @@ -0,0 +1,5 @@ +export { + calculateFleschKincaidGrade, + calculateReadabilityMetrics, + type ReadabilityMetrics, +} from './readability.js'; diff --git a/sdks/typescript/src/features/readability.ts b/sdks/typescript/src/features/readability.ts new file mode 100644 index 0000000..a744cb5 --- /dev/null +++ b/sdks/typescript/src/features/readability.ts @@ -0,0 +1,49 @@ +import nlp from 'compromise'; +import { syllable } from 'syllable'; + +/** + * Calculate Flesch-Kincaid Grade Level + * Equivalent to Python's textstat.flesch_kincaid_grade() + */ +export function calculateFleschKincaidGrade(text: string): number { + return calculateReadabilityMetrics(text).fleschKincaidGrade; +} + +/** + * Additional readability metrics + */ +export interface ReadabilityMetrics { + sentenceCount: number; + wordCount: number; + characterCount: number; + syllableCount: number; + avgWordsPerSentence: number; + avgSyllablesPerWord: number; + fleschKincaidGrade: number; +} + +export function calculateReadabilityMetrics(text: string): ReadabilityMetrics { + const doc = nlp(text); + + const sentences = doc.sentences().length; + const terms = doc.terms(); + const words = terms.length; + const characters = text.replace(/\s/g, '').length; + + const allWords = terms.out('array'); + const totalSyllables = allWords.reduce((sum: number, word: string) => sum + syllable(word), 0); + + const avgWordsPerSentence = sentences > 0 ? words / sentences : 0; + const avgSyllablesPerWord = words > 0 ? totalSyllables / words : 0; + const fkGrade = 0.39 * avgWordsPerSentence + 11.8 * avgSyllablesPerWord - 15.59; + + return { + sentenceCount: sentences, + wordCount: words, + characterCount: characters, + syllableCount: totalSyllables, + avgWordsPerSentence, + avgSyllablesPerWord, + fleschKincaidGrade: Math.round(Math.max(0, fkGrade) * 100) / 100, + }; +} diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index cb0ff5c..16ac0e8 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -1 +1,55 @@ -export {}; +// Core types and schemas +export type { + EvaluationResult, + EvaluationMetadata, + BatchEvaluationResult, + BatchSummary, + EvaluationError, +} from './schemas/index.js'; + +// Error types +export { + EvaluatorError, + ConfigurationError, + ValidationError, + APIError, + AuthenticationError, + RateLimitError, + NetworkError, + TimeoutError, +} from './errors.js'; + +// Logger +export type { Logger, LogContext } from './logger.js'; +export { LogLevel } from './logger.js'; + +// Provider types (for implementing custom providers) +export type { + LLMProvider, + LLMRequest, + LLMResponse, + TextGenerationResponse, + Message, + ProviderConfig, +} from './providers/index.js'; + +// Vocabulary exports +export type { + VocabularyComplexity, + VocabularyComplexityLevel, +} from './schemas/vocabulary.js'; + +export { + VocabularyEvaluator, + evaluateVocabulary, + type VocabularyEvaluatorConfig, + type BaseEvaluatorConfig, + type TelemetryOptions, +} from './evaluators/index.js'; + +// Features +export { + calculateFleschKincaidGrade, + calculateReadabilityMetrics, + type ReadabilityMetrics, +} from './features/index.js'; diff --git a/sdks/typescript/src/logger.ts b/sdks/typescript/src/logger.ts new file mode 100644 index 0000000..bdc5a1b --- /dev/null +++ b/sdks/typescript/src/logger.ts @@ -0,0 +1,159 @@ +/** + * Logging interface for the Evaluators SDK + * + * Provides structured logging with verbosity levels. + * Users can inject custom loggers or use the default console logger. + */ + +/** + * Log levels in order of verbosity + */ +export enum LogLevel { + /** Debug messages - very verbose, for development */ + DEBUG = 0, + /** Informational messages - normal operations */ + INFO = 1, + /** Warning messages - potentially problematic situations */ + WARN = 2, + /** Error messages - errors that need attention */ + ERROR = 3, + /** Silent - no logging */ + SILENT = 4, +} + +/** + * Context object for structured logging + */ +export interface LogContext { + /** Evaluator type (vocabulary, sentence-structure, etc.) */ + evaluator?: string; + /** Current operation or stage */ + operation?: string; + /** Error object if applicable */ + error?: Error; + /** Additional metadata */ + [key: string]: unknown; +} + +/** + * Logger interface + * + * Implement this interface to provide custom logging behavior. + * + * @example + * ```typescript + * const customLogger: Logger = { + * debug: (msg, ctx) => myLogger.debug(msg, ctx), + * info: (msg, ctx) => myLogger.info(msg, ctx), + * warn: (msg, ctx) => myLogger.warn(msg, ctx), + * error: (msg, ctx) => myLogger.error(msg, ctx), + * }; + * + * const evaluator = new VocabularyEvaluator({ + * googleApiKey: '...', + * openaiApiKey: '...', + * logger: customLogger, + * logLevel: LogLevel.INFO, + * }); + * ``` + */ +export interface Logger { + /** + * Log debug message + * Used for detailed debugging information + */ + debug(message: string, context?: LogContext): void; + + /** + * Log informational message + * Used for normal operations + */ + info(message: string, context?: LogContext): void; + + /** + * Log warning message + * Used for potentially problematic situations + */ + warn(message: string, context?: LogContext): void; + + /** + * Log error message + * Used for errors that need attention + */ + error(message: string, context?: LogContext): void; +} + +/** + * Default console logger implementation + */ +class ConsoleLogger implements Logger { + constructor(private level: LogLevel = LogLevel.WARN) {} + + debug(message: string, context?: LogContext): void { + if (this.level <= LogLevel.DEBUG) { + console.debug(`[DEBUG] ${message}`, context || ''); + } + } + + info(message: string, context?: LogContext): void { + if (this.level <= LogLevel.INFO) { + console.info(`[INFO] ${message}`, context || ''); + } + } + + warn(message: string, context?: LogContext): void { + if (this.level <= LogLevel.WARN) { + console.warn(`[WARN] ${message}`, context || ''); + } + } + + error(message: string, context?: LogContext): void { + if (this.level <= LogLevel.ERROR) { + console.error(`[ERROR] ${message}`, context || ''); + } + } +} + +/** + * Silent logger - logs nothing + */ +class SilentLogger implements Logger { + debug(): void {} + info(): void {} + warn(): void {} + error(): void {} +} + +/** + * Create a logger instance + * + * @param customLogger - Optional custom logger implementation + * @param level - Log level (default: WARN) + * @returns Logger instance + */ +export function createLogger(customLogger?: Logger, level: LogLevel = LogLevel.WARN): Logger { + // Use custom logger if provided + if (customLogger) { + return customLogger; + } + + // Use silent logger if level is SILENT + if (level === LogLevel.SILENT) { + return new SilentLogger(); + } + + // Use console logger with specified level + return new ConsoleLogger(level); +} + +/** + * Format error for logging + * + * @internal + */ +export function formatError(error: unknown): string { + if (error instanceof Error) { + return `${error.name}: ${error.message}`; + } + return String(error); +} diff --git a/sdks/typescript/src/prompts/vocabulary/background-knowledge.ts b/sdks/typescript/src/prompts/vocabulary/background-knowledge.ts new file mode 100644 index 0000000..52309f1 --- /dev/null +++ b/sdks/typescript/src/prompts/vocabulary/background-knowledge.ts @@ -0,0 +1,10 @@ +import BACKGROUND_KNOWLEDGE_TEMPLATE from '../../../../../evals/prompts/vocabulary/background-knowledge.txt'; + +/** + * Generate the background knowledge prompt for a given text and grade level + */ +export function getBackgroundKnowledgePrompt(text: string, grade: string): string { + return BACKGROUND_KNOWLEDGE_TEMPLATE + .replaceAll('{grade}', grade) + .replaceAll('{text}', text); +} diff --git a/sdks/typescript/src/prompts/vocabulary/index.ts b/sdks/typescript/src/prompts/vocabulary/index.ts new file mode 100644 index 0000000..47ed85a --- /dev/null +++ b/sdks/typescript/src/prompts/vocabulary/index.ts @@ -0,0 +1,3 @@ +export { getBackgroundKnowledgePrompt } from './background-knowledge.js'; +export { getSystemPrompt } from './system.js'; +export { getUserPrompt } from './user.js'; diff --git a/sdks/typescript/src/prompts/vocabulary/system.ts b/sdks/typescript/src/prompts/vocabulary/system.ts new file mode 100644 index 0000000..81dde16 --- /dev/null +++ b/sdks/typescript/src/prompts/vocabulary/system.ts @@ -0,0 +1,17 @@ +import SYSTEM_PROMPT_GRADES_3_4 from '../../../../../evals/prompts/vocabulary/grades-3-4-system.txt'; +import SYSTEM_PROMPT_OTHER_GRADES from '../../../../../evals/prompts/vocabulary/other-grades-system.txt'; + +/** + * Get the appropriate system prompt based on grade level + * @param grade - The target grade level (3-12) + * @returns The system prompt for the grade level + */ +export function getSystemPrompt(grade: string): string { + // Grades 3-4 use the GRADES_3_4 prompt + if (grade === '3' || grade === '4') { + return SYSTEM_PROMPT_GRADES_3_4; + } + + // All other grades (5-12) use OTHER_GRADES prompt + return SYSTEM_PROMPT_OTHER_GRADES; +} diff --git a/sdks/typescript/src/prompts/vocabulary/user.ts b/sdks/typescript/src/prompts/vocabulary/user.ts new file mode 100644 index 0000000..75e56b0 --- /dev/null +++ b/sdks/typescript/src/prompts/vocabulary/user.ts @@ -0,0 +1,28 @@ +import USER_PROMPT_TEMPLATE_GRADES_3_4 from '../../../../../evals/prompts/vocabulary/grades-3-4-user.txt'; +import USER_PROMPT_TEMPLATE_OTHER_GRADES from '../../../../../evals/prompts/vocabulary/other-grades-user.txt'; + +/** + * Generate the user prompt for vocabulary complexity evaluation + * @param text - The text to evaluate + * @param studentGradeLevel - The student's grade level + * @param studentBackgroundKnowledge - Background knowledge assumption + * @param fkLevel - Flesch-Kincaid grade level + * @returns The formatted user prompt + */ +export function getUserPrompt( + text: string, + studentGradeLevel: string, + studentBackgroundKnowledge: string, + fkLevel: number +): string { + // Select the appropriate template based on grade + const template = studentGradeLevel === '3' || studentGradeLevel === '4' + ? USER_PROMPT_TEMPLATE_GRADES_3_4 + : USER_PROMPT_TEMPLATE_OTHER_GRADES; + + return template + .replaceAll('{student_grade_level}', studentGradeLevel) + .replaceAll('{student_background_knowledge}', studentBackgroundKnowledge) + .replaceAll('{fk_level}', fkLevel.toString()) + .replaceAll('{text}', text); +} diff --git a/sdks/typescript/src/providers/ai-sdk-provider.ts b/sdks/typescript/src/providers/ai-sdk-provider.ts new file mode 100644 index 0000000..e482f35 --- /dev/null +++ b/sdks/typescript/src/providers/ai-sdk-provider.ts @@ -0,0 +1,144 @@ +import { generateText as aiGenerateText, Output } from 'ai'; +import type { + LLMProvider, + LLMRequest, + LLMResponse, + Message, + ProviderConfig, +} from './base.js'; + +/** + * Default models for each provider based on Python implementation + */ +const DEFAULT_MODELS = { + openai: 'gpt-4o', + anthropic: 'claude-sonnet-4-5-20250929', + google: 'gemini-2.5-pro', +} as const; + +/** + * Vercel AI SDK provider implementation + * Supports OpenAI, Anthropic, and Google Gemini + */ +export class VercelAIProvider implements LLMProvider { + constructor(private config: ProviderConfig) { + if (config.type === 'custom') { + throw new Error( + 'VercelAIProvider does not support custom type. Use config.customProvider directly.' + ); + } + } + + /** + * Generate structured output using Vercel AI SDK's generateText with output + */ + async generateStructured(request: LLMRequest): Promise> { + const model = await this.getModel(request.model); + const startTime = Date.now(); + + const { output, usage } = await aiGenerateText({ + model, + messages: request.messages, + output: Output.object({ schema: request.schema }), + temperature: request.temperature ?? 0, + maxRetries: this.config.maxRetries ?? 0, + ...(request.maxTokens !== undefined ? { maxTokens: request.maxTokens } : {}), + }); + + return { + data: output as T, + model: request.model || this.getDefaultModel(), + usage: { + inputTokens: usage.inputTokens || 0, + outputTokens: usage.outputTokens || 0, + }, + latencyMs: Date.now() - startTime, + }; + } + + /** + * Generate plain text using Vercel AI SDK's generateText + */ + async generateText(messages: Message[], temperature?: number): Promise { + const model = await this.getModel(); + const startTime = Date.now(); + + const { text, usage } = await aiGenerateText({ + model, + messages, + temperature: temperature ?? this.config.temperature ?? 0, + maxRetries: this.config.maxRetries ?? 0, + }); + + return { + text, + usage: { + inputTokens: usage.inputTokens || 0, + outputTokens: usage.outputTokens || 0, + }, + latencyMs: Date.now() - startTime, + }; + } + + /** + * Get the configured language model. + * Uses dynamic imports so consumers only need to install the provider packages they use. + */ + private async getModel(requestModel?: string) { + const modelId = requestModel || this.config.model || this.getDefaultModel(); + const apiKey = this.config.apiKey; + + switch (this.config.type) { + case 'openai': { + const { createOpenAI } = await import('@ai-sdk/openai').catch(() => { + throw new Error( + 'To use the OpenAI provider, install its adapter: npm install @ai-sdk/openai' + ); + }); + return createOpenAI(apiKey ? { apiKey } : {})(modelId); + } + case 'anthropic': { + const { createAnthropic } = await import('@ai-sdk/anthropic').catch(() => { + throw new Error( + 'To use the Anthropic provider, install its adapter: npm install @ai-sdk/anthropic' + ); + }); + return createAnthropic(apiKey ? { apiKey } : {})(modelId); + } + case 'google': { + const { createGoogleGenerativeAI } = await import('@ai-sdk/google').catch(() => { + throw new Error( + 'To use the Google provider, install its adapter: npm install @ai-sdk/google' + ); + }); + return createGoogleGenerativeAI(apiKey ? { apiKey } : {})(modelId); + } + default: + throw new Error(`Unsupported provider type: ${this.config.type}`); + } + } + + /** + * Get default model for the configured provider + */ + private getDefaultModel(): string { + const providerType = this.config.type; + + if (providerType === 'custom') { + throw new Error('Cannot get default model for custom provider type'); + } + + return DEFAULT_MODELS[providerType]; + } +} + +/** + * Factory function to create a provider instance + */ +export function createProvider(config: ProviderConfig): LLMProvider { + if (config.type === 'custom' && config.customProvider) { + return config.customProvider; + } + + return new VercelAIProvider(config); +} diff --git a/sdks/typescript/src/providers/base.ts b/sdks/typescript/src/providers/base.ts new file mode 100644 index 0000000..5b6dee9 --- /dev/null +++ b/sdks/typescript/src/providers/base.ts @@ -0,0 +1,73 @@ +import type { z } from 'zod'; + +/** + * Message format for LLM conversations + */ +export interface Message { + role: 'system' | 'user' | 'assistant'; + content: string; +} + +/** + * Request configuration for structured LLM generation + */ +export interface LLMRequest { + messages: Message[]; + schema: z.ZodSchema; + temperature?: number; + maxTokens?: number; + model?: string; +} + +/** + * Response from LLM with usage metadata + */ +export interface LLMResponse { + data: T; + model: string; + usage: { + inputTokens: number; + outputTokens: number; + }; + latencyMs: number; +} + +/** + * Response from plain text generation + */ +export interface TextGenerationResponse { + text: string; + usage: { + inputTokens: number; + outputTokens: number; + }; + latencyMs: number; +} + +/** + * Base interface for LLM provider implementations + */ +export interface LLMProvider { + /** + * Generate structured output from LLM using Zod schema + */ + generateStructured(request: LLMRequest): Promise>; + + /** + * Generate plain text from LLM + */ + generateText(messages: Message[], temperature?: number): Promise; +} + +/** + * Configuration for LLM provider + */ +export interface ProviderConfig { + type: 'openai' | 'anthropic' | 'google' | 'custom'; + apiKey?: string; + model?: string; + temperature?: number; + baseURL?: string; + customProvider?: LLMProvider; + maxRetries?: number; +} diff --git a/sdks/typescript/src/providers/index.ts b/sdks/typescript/src/providers/index.ts new file mode 100644 index 0000000..f32e5e3 --- /dev/null +++ b/sdks/typescript/src/providers/index.ts @@ -0,0 +1,10 @@ +export type { + LLMProvider, + LLMRequest, + LLMResponse, + TextGenerationResponse, + Message, + ProviderConfig, +} from './base.js'; + +export { VercelAIProvider, createProvider } from './ai-sdk-provider.js'; diff --git a/sdks/typescript/src/schemas/index.ts b/sdks/typescript/src/schemas/index.ts new file mode 100644 index 0000000..ded6c72 --- /dev/null +++ b/sdks/typescript/src/schemas/index.ts @@ -0,0 +1,10 @@ +export { + ComplexityLevel, + GradeLevel, + type EvaluationResult, + type EvaluationMetadata, + type BatchEvaluationResult, + type BatchSummary, + type EvaluationError, +} from './outputs.js'; + diff --git a/sdks/typescript/src/schemas/outputs.ts b/sdks/typescript/src/schemas/outputs.ts new file mode 100644 index 0000000..9ab807e --- /dev/null +++ b/sdks/typescript/src/schemas/outputs.ts @@ -0,0 +1,75 @@ +import { z } from 'zod'; + +/** + * Complexity levels for sentence structure evaluation + */ +export const ComplexityLevel = z.enum([ + 'Slightly Complex', + 'Moderately Complex', + 'Very Complex', + 'Exceedingly Complex', +]); + +export type ComplexityLevel = z.infer; + +/** + * Grade levels for vocabulary evaluation + */ +export const GradeLevel = z.enum([ + 'Below Grade Level', + 'At Grade Level', + 'Above Grade Level', +]); + +export type GradeLevel = z.infer; + +/** + * Metadata attached to all evaluation results + */ +export interface EvaluationMetadata { + evaluatorVersion?: string; + promptVersion: string; + model: string; + timestamp: Date; + processingTimeMs: number; +} + +/** + * Base evaluation result structure + */ +export interface EvaluationResult { + score: TScore; + reasoning: string; + metadata: EvaluationMetadata; + _internal?: TInternal; +} + +/** + * Batch evaluation summary statistics + */ +export interface BatchSummary { + total: number; + successful: number; + failed: number; + averageProcessingTimeMs: number; +} + +/** + * Error type for failed evaluations + */ +export interface EvaluationError { + error: string; + input: { + text: string; + grade?: string; + }; + timestamp: Date; +} + +/** + * Batch evaluation result + */ +export interface BatchEvaluationResult { + results: Array; + summary: BatchSummary; +} diff --git a/sdks/typescript/src/schemas/vocabulary.ts b/sdks/typescript/src/schemas/vocabulary.ts new file mode 100644 index 0000000..f5b80d0 --- /dev/null +++ b/sdks/typescript/src/schemas/vocabulary.ts @@ -0,0 +1,39 @@ +import { z } from 'zod'; + +/** + * Vocabulary complexity levels matching Qual Text Complexity rubric (SAP) + */ +export const VocabularyComplexityLevel = z.enum([ + 'slightly complex', + 'moderately complex', + 'very complex', + 'exceedingly complex', +]); + +export type VocabularyComplexityLevel = z.infer; + +/** + * Vocabulary complexity evaluation output + * Ported from Python Output BaseModel + */ +export const VocabularyComplexitySchema = z.object({ + tier_2_words: z.string().describe('List of Tier 2 words (academic words)'), + tier_3_words: z.string().describe('List of Tier 3 words (domain-specific)'), + archaic_words: z.string().describe('List of Archaic words'), + other_complex_words: z.string().describe('List of Other Complex words'), + complexity_score: VocabularyComplexityLevel.describe( + 'The complexity of the text vocabulary' + ), + reasoning: z.string().describe('Detailed reasoning for the complexity rating'), +}); + +export type VocabularyComplexity = z.infer; + +/** + * Background knowledge assumption for a student at a given grade level + * This is generated in Stage 1 and used as input for Stage 2 + */ +export interface BackgroundKnowledge { + assumption: string; + grade: string; +} diff --git a/sdks/typescript/src/telemetry/client.ts b/sdks/typescript/src/telemetry/client.ts new file mode 100644 index 0000000..d4db550 --- /dev/null +++ b/sdks/typescript/src/telemetry/client.ts @@ -0,0 +1,64 @@ +import type { TelemetryConfig, TelemetryEvent } from './types.js'; +import type { Logger } from '../logger.js'; + +/** + * Telemetry client for sending analytics events + * + * Fire-and-forget implementation that never blocks SDK operations. + * Errors are logged but don't fail evaluations. + */ +export class TelemetryClient { + private config: TelemetryConfig; + private logger: Logger; + + constructor(config: TelemetryConfig) { + this.config = config; + this.logger = config.logger; + } + + /** + * Send telemetry event to analytics service + * + * Fire-and-forget: Errors are logged but don't throw. + */ + async send(event: TelemetryEvent): Promise { + // Skip if telemetry disabled + if (!this.config.enabled) { + return; + } + + try { + const headers: Record = { + 'Content-Type': 'application/json', + 'X-Client-ID': this.config.clientId, + }; + + // Add partner key if provided + if (this.config.partnerKey) { + headers['X-API-Key'] = this.config.partnerKey; + } + + const response = await fetch(this.config.endpoint, { + method: 'POST', + headers, + body: JSON.stringify(event), + // Don't block SDK operations on slow networks + signal: AbortSignal.timeout(5000), // 5 second timeout + }); + + if (!response.ok) { + this.logger.warn( + `[Telemetry] Failed to send event: ${response.status} ${response.statusText}` + ); + } + } catch (error) { + // Log error but never throw (fire-and-forget) + if (error instanceof Error) { + // Don't log timeout errors (expected on slow networks) + if (error.name !== 'TimeoutError' && error.name !== 'AbortError') { + this.logger.warn(`[Telemetry] Error sending event: ${error.message}`); + } + } + } + } +} diff --git a/sdks/typescript/src/telemetry/index.ts b/sdks/typescript/src/telemetry/index.ts new file mode 100644 index 0000000..ae1cbb7 --- /dev/null +++ b/sdks/typescript/src/telemetry/index.ts @@ -0,0 +1,10 @@ +export { TelemetryClient } from './client.js'; +export { generateClientId, getSDKVersion } from './utils.js'; +export type { + TelemetryConfig, + TelemetryEvent, + EvaluationStatus, + TokenUsage, + StageDetail, + TelemetryMetadata, +} from './types.js'; diff --git a/sdks/typescript/src/telemetry/types.ts b/sdks/typescript/src/telemetry/types.ts new file mode 100644 index 0000000..31b2920 --- /dev/null +++ b/sdks/typescript/src/telemetry/types.ts @@ -0,0 +1,92 @@ +// TODO: Generate these types from the telemetry service OpenAPI/JSON Schema +// instead of maintaining them manually. This will prevent drift between +// client and server schemas. + +/** + * Evaluation status + */ +export type EvaluationStatus = 'success' | 'error'; + +/** + * Token usage metrics from LLM providers + */ +export interface TokenUsage { + input_tokens: number; + output_tokens: number; +} + +/** + * Per-stage details for multi-stage evaluations + */ +export interface StageDetail { + /** Stage name (e.g., "background_knowledge", "complexity_evaluation") */ + stage: string; + + /** Provider used for this stage (e.g., "openai:gpt-4o") */ + provider: string; + + /** Total latency including all retries (ms) */ + latency_ms: number; + + /** Token usage aggregated across all attempts */ + token_usage?: TokenUsage; + + /** + * Whether schema validation failed (indicates prompt needs clearer instructions) + * + * TODO: Not currently tracked. Vercel AI SDK abstracts validation away. + * To implement: Add custom retry wrapper that catches validation errors. + */ + schema_validation_failed?: boolean; +} + +/** + * Extensible metadata for telemetry events + */ +export interface TelemetryMetadata { + /** Detailed breakdown by stage (for multi-stage evaluations) */ + stage_details?: StageDetail[]; + + // Future fields can be added here: + // cache_hit?: boolean; + // prompt_tokens_breakdown?: {...}; + // etc. +} + +/** + * Telemetry event payload + */ +export interface TelemetryEvent { + timestamp: string; + sdk_version: string; + evaluator_type: string; + grade?: string; + status: EvaluationStatus; + error_code?: string; + latency_ms: number; + text_length_chars: number; + provider: string; // Format: "provider:model" or "provider1+provider2" for multi-provider + token_usage?: TokenUsage; // Aggregated across all stages and attempts + metadata?: TelemetryMetadata; // Optional per-stage breakdown + input_text?: string; // Input text (only if recordInputs enabled) +} + +/** + * Configuration for telemetry client + */ +export interface TelemetryConfig { + /** Analytics service endpoint URL */ + endpoint: string; + + /** Learning Commons partner key (optional, sent as X-API-Key header) */ + partnerKey?: string; + + /** Client ID for anonymous tracking (persistent UUID from ~/.config/learning-commons/config.json) */ + clientId: string; + + /** Enable telemetry (default: true) */ + enabled: boolean; + + /** Logger instance (respects the SDK's configured log level and custom logger) */ + logger: import('../logger.js').Logger; +} diff --git a/sdks/typescript/src/telemetry/utils.ts b/sdks/typescript/src/telemetry/utils.ts new file mode 100644 index 0000000..eaef3d9 --- /dev/null +++ b/sdks/typescript/src/telemetry/utils.ts @@ -0,0 +1,93 @@ +import { randomUUID } from 'node:crypto'; +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { homedir } from 'node:os'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +/** Cached client ID — populated on first call, reused for process lifetime */ +let cachedClientId: string | undefined; + +/** + * Get or create a persistent client ID for anonymous tracking. + * + * On first run, generates a UUID and tries to save it to: + * - Windows: %APPDATA%\learning-commons\config.json + * - macOS/Linux: ~/.config/learning-commons/config.json + * + * On subsequent runs, reads the saved UUID from disk. + * Falls back to an in-memory UUID (per-process) if the filesystem + * is unavailable (e.g., serverless, read-only containers). + */ +export function generateClientId(): string { + if (cachedClientId) { + return cachedClientId; + } + + const configFile = getConfigFilePath(); + + // Try to read existing client ID from disk + try { + const data = JSON.parse(readFileSync(configFile, 'utf-8')) as { + telemetry?: { clientId?: string }; + }; + if (data?.telemetry?.clientId) { + cachedClientId = data.telemetry.clientId; + return cachedClientId; + } + } catch { + // File doesn't exist yet — fall through to generate + } + + // Generate new UUID and try to persist it + const clientId = randomUUID(); + try { + mkdirSync(dirname(configFile), { recursive: true }); + writeFileSync(configFile, JSON.stringify({ telemetry: { clientId } }, null, 2)); + } catch { + // Filesystem unavailable — use in-memory UUID for this process + } + + cachedClientId = clientId; + return cachedClientId; +} + +function getConfigFilePath(): string { + const configDir = + process.platform === 'win32' + ? join(process.env.APPDATA ?? homedir(), 'learning-commons') + : join(homedir(), '.config', 'learning-commons'); + return join(configDir, 'config.json'); +} + +let cachedVersion: string | undefined; + +/** + * Get SDK version from package.json + */ +export function getSDKVersion(): string { + if (cachedVersion) { + return cachedVersion; + } + + const possiblePaths = [ + join(__dirname, '../../package.json'), // From src/ + join(__dirname, '../package.json'), // From dist/ + ]; + + for (const path of possiblePaths) { + try { + const pkg = JSON.parse(readFileSync(path, 'utf-8')) as { version?: string }; + cachedVersion = pkg.version || '0.0.0'; + return cachedVersion; + } catch { + continue; + } + } + + // Fallback if no package.json found + cachedVersion = '0.0.0'; + return cachedVersion; +} diff --git a/sdks/typescript/tests/README.md b/sdks/typescript/tests/README.md new file mode 100644 index 0000000..0f06c3e --- /dev/null +++ b/sdks/typescript/tests/README.md @@ -0,0 +1,221 @@ +# Test Suite + +This directory contains unit and integration tests for the Evaluators SDK. + +## Structure + +``` +tests/ +├── unit/ # Fast tests, no API calls +├── integration/ # Real API calls +└── utils/ # Shared test utilities +``` + +## Running Tests + +### Unit Tests +```bash +npm run test:unit # Fast, no API keys needed +``` + +### Integration Tests +Requires API keys in `.env`: +```bash +OPENAI_API_KEY=sk-... +GOOGLE_API_KEY=... +``` + +Run tests: +```bash +RUN_INTEGRATION_TESTS=true npm run test:integration +``` + +### All Tests +```bash +RUN_INTEGRATION_TESTS=true npm run test:all +``` + +### CI Tests +```bash +npm run test:ci # Tests built dist/ package +``` + +## Key Patterns + +### 1. Acceptable Values for LLM Non-Determinism + +LLMs are non-deterministic. Tests use **expected** values with **acceptable** adjacent values: + +```typescript +{ + id: 'V3', + grade: '3', + text: 'Sample text...', + expected: 'very complex', // Try to match this first + acceptable: ['moderately complex'], // Accept if no expected match +} +``` + +**Strategy:** +- Try up to 3 attempts to match expected value (short-circuit on match) +- If no expected match, check if any result is in acceptable range +- Pass test if either expected or acceptable match found + +### 2. Parallel Test Execution + +All tests run concurrently using `it.concurrent()`: + +```typescript +describeIntegration.concurrent('Test Suite', () => { + TEST_CASES.forEach((testCase) => { + it.concurrent(`${testCase.id}`, async () => { + // Test runs in parallel + }, TEST_TIMEOUT_MS); + }); +}); +``` + +**Benefits**: 3-4x faster test execution + +### 3. Buffered Logging + +Logs are buffered and printed atomically to prevent interleaving: + +```typescript +const logBuffer: string[] = []; +logBuffer.push('Test output...'); +// ... collect all logs +console.log(logBuffer.join('\n')); // Print once at end +``` + +## Writing New Integration Tests + +### Basic Template + +```typescript +import { describe, it, expect, beforeAll } from 'vitest'; +import { MyEvaluator } from '../../src/evaluators/my-evaluator.js'; +import { runEvaluatorTest, type BaseTestCase } from '../utils/index.js'; +import { config } from 'dotenv'; + +config(); + +const SKIP_INTEGRATION = !process.env.RUN_INTEGRATION_TESTS && !process.env.MY_API_KEY; +const describeIntegration = SKIP_INTEGRATION ? describe.skip : describe; +const TEST_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes + +const TEST_CASES: BaseTestCase[] = [ + { + id: 'TEST1', + grade: '3', // Optional, if evaluator needs it + text: 'Sample text...', + expected: 'expected result', + acceptable: ['acceptable alternative'], + }, +]; + +describeIntegration.concurrent('My Evaluator - Test Suite', () => { + let evaluator: MyEvaluator; + + beforeAll(() => { + if (SKIP_INTEGRATION) { + console.log('⏭️ Skipping integration tests'); + return; + } + + evaluator = new MyEvaluator({ + partnerKey: process.env.MY_PARTNER_KEY!, + retry: false, // We handle retries in test logic + }); + }); + + TEST_CASES.forEach((testCase) => { + it.concurrent(`${testCase.id}: ${testCase.expected}`, async () => { + const logBuffer: string[] = []; + + logBuffer.push('\n' + '='.repeat(80)); + logBuffer.push(`Test Case ${testCase.id}`); + logBuffer.push('='.repeat(80)); + + const maxAttempts = 3; + const result = await runEvaluatorTest(testCase, { + evaluator, + extractResult: (r) => r.score, // Extract the field to compare + maxAttempts, + }); + + logBuffer.push(...result.logs); + console.log(logBuffer.join('\n')); + + expect(result.matched).toBe(true); + expect(result.matchedOnAttempt).toBeLessThanOrEqual(maxAttempts); + }, TEST_TIMEOUT_MS); + }); +}); +``` + +### Test Configuration + +```typescript +// Test timeout (2 minutes per test) +const TEST_TIMEOUT_MS = 2 * 60 * 1000; + +// Max retry attempts +const maxAttempts = 3; + +// Skip integration tests if no API keys +const SKIP_INTEGRATION = !process.env.RUN_INTEGRATION_TESTS && !process.env.API_KEY; +``` + +## Test Utilities + +### `runEvaluatorTest(testCase, config)` + +Generic test runner for all evaluators: + +```typescript +const result = await runEvaluatorTest(testCase, { + evaluator: myEvaluator, + extractResult: (r) => r.score, // How to extract result from evaluation + maxAttempts: 3, // Default: 3 +}); + +// Result structure +interface TestResult { + matched: boolean; // Did test pass? + matchedOnAttempt?: number; // Which attempt matched? + matchType?: 'expected' | 'acceptable'; // How did it match? + totalAttempts: number; + allResults: string[]; // All attempt results + logs: string[]; // Buffered log messages +} +``` + +## Test Strategy + +### Local Development +Tests run against `src/` with prompts copied from `../../evals/prompts/`: +```bash +npm run test:unit +npm run test:integration +``` + +### CI/CD +Tests run against built `dist/` package to validate published code: +```bash +npm run test:ci +``` + +## Troubleshooting + +**Tests skipped?** +- Check API keys: `echo $OPENAI_API_KEY` +- Set: `RUN_INTEGRATION_TESTS=true npm run test:integration` + +**Tests timeout?** +- Increase `TEST_TIMEOUT_MS = 3 * 60 * 1000` (3 minutes) + +**Tests flaky?** +- Add more acceptable values based on actual LLM output +- Increase `maxAttempts` from 3 to 5 +- Check if test case is ambiguous diff --git a/sdks/typescript/tests/integration/vocabulary.integration.test.ts b/sdks/typescript/tests/integration/vocabulary.integration.test.ts new file mode 100644 index 0000000..49b4662 --- /dev/null +++ b/sdks/typescript/tests/integration/vocabulary.integration.test.ts @@ -0,0 +1,141 @@ +import { describe, it, expect, beforeAll } from 'vitest'; +import { VocabularyEvaluator } from '../../src/evaluators/vocabulary.js'; +import { + runEvaluatorTest, + type BaseTestCase, +} from '../utils/index.js'; + +/** + * Vocabulary Evaluator Integration Tests + * + * Test cases cover grades 3-9 with varying complexity levels. + * + * Each test uses a retry mechanism (up to 3 attempts) to account for LLM non-determinism, + * with short-circuiting on first expected match. If no expected match is found after all + * attempts, the test checks if any result falls within the acceptable value range. + * + * To run these tests: + * ```bash + * RUN_INTEGRATION_TESTS=true npm run test:integration + * ``` + */ + +const SKIP_INTEGRATION = !process.env.RUN_INTEGRATION_TESTS && + (!process.env.OPENAI_API_KEY || !process.env.GOOGLE_API_KEY); + +const describeIntegration = SKIP_INTEGRATION ? describe.skip : describe; + +// Test timeout: 2 minutes per test case (allows for 3 attempts with API latency) +const TEST_TIMEOUT_MS = 2 * 60 * 1000; + +// Test cases from PR #6 +const TEST_CASES: BaseTestCase[] = [ + { + id: 'V3', + grade: '3', + text: 'Civil rights are rights that all people in a country have. The civil rights of a country apply to all the citizens within its borders. These rights are given by the laws of the country. Civil rights are sometimes thought to be the same as natural rights. In many countries civil rights include freedom of speech, freedom of the press, freedom of religion, and freedom of assembly. Civil rights also include the right to own property and the right to get fair and equal treatment from the government, from other citizens, and from private groups.', + expected: 'very complex', + acceptable: ['moderately complex', 'exceedingly complex'], + }, + { + id: 'V4', + grade: '4', + text: 'Bluetooth is a protocol for wireless communication over short distances. It was developed in the 1990s, to reduce the number of cables. Devices such as mobile phones, laptops, PCs, printers, digital cameras and video game consoles can connect to each other, and exchange information. This is done using radio waves. It can be done securely. Bluetooth is only used for relatively short distances, like a few metres. There are different standards. Data rates vary. Currently, they are at 1-3 MBit per second.', + expected: 'exceedingly complex', + acceptable: ['very complex'], + }, + { + id: 'V5', + grade: '5', + text: `The scientific method is a way to learn about the world around us. It helps us figure out how things work. Scientists use the scientific method to test their ideas. They start by making observations and asking questions. Then, they make a guess, or a hypothesis, about what might be the answer. They use their hypothesis to make predictions about what will happen in an experiment. Scientists then test their predictions by doing experiments. If the results of the experiment match their predictions, then their hypothesis is supported. If the results don't match, then they need to change their hypothesis. Scientists repeat this process many times to make sure their hypothesis is correct. The scientific method is important because it helps us learn new things. It helps us understand the world around us. Scientists use the scientific method to make new discoveries and solve problems.`, + expected: 'slightly complex', + acceptable: ['moderately complex'], + }, + { + id: 'V6', + grade: '6', + text: `Chicago in 1871 was a city ready to burn. The city boasted having 59,500 buildings, many of them—such as the Courthouse and the Tribune Building—large and ornately decorated. The trouble was that about two-thirds of all these structures were made entirely of wood. Many of the remaining buildings (even the ones proclaimed to be 'fireproof') looked solid, but were actually jerrybuilt affairs; the stone or brick exteriors hid wooden frames and floors, all topped with highly flammable tar or shingle roofs. It was also a common practice to disguise wood as another kind of building material. The fancy exterior decorations on just about every building were carved from wood, then painted to look like stone or marble.`, + expected: 'very complex', + acceptable: ['moderately complex', 'exceedingly complex'], + }, + { + id: 'V7', + grade: '7', + text: `The scientific method is a way of learning about the world around us. It's a process that helps us understand how things work and why they happen. It's not just for scientists; we all use the scientific method in our everyday lives, even if we don't realize it. The scientific method starts with an observation. We notice something interesting and want to know more about it. For example, you might notice that your plant is wilting. You might wonder why this is happening. Next, we form a hypothesis, which is a possible explanation for our observation. In our plant example, you might hypothesize that the plant is wilting because it needs more water. Then, we test our hypothesis by doing an experiment. We change something in our experiment to see if it affects the outcome. In our plant example, you could water the plant and see if it recovers. Based on the results of our experiment, we can either support or reject our hypothesis. If the plant recovers after being watered, then your hypothesis is supported. If the plant doesn't recover, then you need to come up with a new hypothesis. The scientific method is a powerful tool for learning and understanding the world around us. It's a process of asking questions, testing ideas, and drawing conclusions based on evidence. It's a way of thinking that helps us to be curious, to be critical, and to be open to new ideas.`, + expected: 'slightly complex', + acceptable: ['moderately complex'], + }, + { + id: 'V8', + grade: '8', + text: 'The American Revolution was a war for independence between the thirteen American colonies and Great Britain. The war started in 1775 and ended in 1783. The colonists wanted to be free from British rule. They wanted to make their own laws and govern themselves. The colonists were angry about new taxes that the British Parliament imposed on them. They felt that they were being taxed without having a say in how the money was spent. The colonists also felt that the British government was not treating them fairly. The war began with the Battles of Lexington and Concord in April 1775. The colonists, led by General George Washington, fought against the British army. The war was long and difficult, but the colonists eventually won. The colonists won the war because they had the support of the French. The French helped the colonists by providing them with soldiers, ships, and money. The colonists also had a strong leader in George Washington. He was a skilled military leader and he inspired the colonists to fight for their freedom. The American Revolution was a turning point in history. It showed that colonies could break free from their mother countries and become independent nations. The American Revolution also inspired other revolutions around the world.', + expected: 'slightly complex', + acceptable: ['moderately complex'], + }, + { + id: 'V9', + grade: '9', + text: `Mr. President: I would like to speak briefly and simply about a serious national condition. It is a national feeling of fear and frustration that could result in national suicide and the end of everything that we Americans hold dear. It is a condition that comes from the lack of effective leadership in either the Legislative Branch or the Executive Branch of our Government. That leadership is so lacking that serious and responsible proposals are being made that national advisory commissions be appointed to provide such critically needed leadership. I speak as briefly as possible because too much harm has already been done with irresponsible words of bitterness and selfish political opportunism. I speak as briefly as possible because the issue is too great to be obscured by eloquence. I speak simply and briefly in the hope that my words will be taken to heart. I speak as a Republican. I speak as a woman. I speak as a United States Senator. I speak as an American. The United States Senate has long enjoyed worldwide respect as the greatest deliberative body in the world. But recently that deliberative character has too often been debased to the level of a forum of hate and character assassination sheltered by the shield of congressional immunity. It is ironical that we Senators can in debate in the Senate directly or indirectly, by any form of words, impute to any American who is not a Senator any conduct or motive unworthy or unbecoming an American—and without that non-Senator American having any legal redress against us—yet if we say the same thing in the Senate about our colleagues we can be stopped on the grounds of being out of order. It is strange that we can verbally attack anyone else without restraint and with full protection and yet we hold ourselves above the same type of criticism here on the Senate Floor. Surely the United States Senate is big enough to take self-criticism and self-appraisal. Surely we should be able to take the same kind of character attacks that we "dish out" to outsiders. I think that it is high time for the United States Senate and its members to do some soul-searching—for us to weigh our consciences—on the manner in which we are performing our duty to the people of America—on the manner in which we are using or abusing our individual powers and privileges. I think that it is high time that we remembered that we have sworn to uphold and defend the Constitution. I think that it is high time that we remembered that the Constitution, as amended, speaks not only of the freedom of speech but also of trial by jury instead of trial by accusation. Whether it be a criminal prosecution in court or a character prosecution in the Senate, there is little practical distinction when the life of a person has been ruined. Those of us who shout the loudest about Americanism in making character assassinations are all too frequently those who, by our own words and acts, ignore some of the basic principles of Americanism: The right to criticize; The right to hold unpopular beliefs; The right to protest; The right of independent thought. The exercise of these rights should not cost one single American citizen his reputation or his right to a livelihood nor should he be in danger of losing his reputation or livelihood merely because he happens to know someone who holds unpopular beliefs. Who of us doesn't? Otherwise none of us could call our souls our own. Otherwise thought control would have set in. The American people are sick and tired of being afraid to speak their minds lest they be politically smeared as "Communists" or "Fascists" by their opponents. Freedom of speech is not what it used to be in America. It has been so abused by some that it is not exercised by others. The American people are sick and tired of seeing innocent people smeared and guilty people whitewashed. But there have been enough proved cases, such as the Amerasia case, the Hiss case, the Coplon case, the Gold case, to cause the nationwide distrust and strong suspicion that there may be something to the unproved, sensational accusations. I doubt if the Republican Party could—simply because I don't believe the American people will uphold any political party that puts political exploitation above national interest. Surely we Republicans aren't that desperate for victory. I don't want to see the Republican Party win that way. While it might be a fleeting victory for the Republican Party, it would be a more lasting defeat for the American people. Surely it would ultimately be suicide for the Republican Party and the two-party system that has protected our American liberties from the dictatorship of a one-party system. As members of the Minority Party, we do not have the primary authority to formulate the policy of our Government. But we do have the responsibility of rendering constructive criticism, of clarifying issues, of allaying fears by acting as responsible citizens. As a woman, I wonder how the mothers, wives, sisters, and daughters feel about the way in which members of their families have been politically mangled in the Senate debate—and I use the word "debate" advisedly. As a United States Senator, I am not proud of the way in which the Senate has been made a publicity platform for irresponsible sensationalism. I am not proud of the reckless abandon in which unproved charges have been hurled from the side of the aisle. I am not proud of the obviously staged, undignified countercharges that have been attempted in retaliation from the other side of the aisle. I don't like the way the Senate has been made a rendezvous for vilification, for selfish political gain at the sacrifice of individual reputations and national unity. I am not proud of the way we smear outsiders from the Floor of the Senate and hide behind the cloak of congressional immunity and still place ourselves beyond criticism on the Floor of the Senate. As an American, I am shocked at the way Republicans and Democrats alike are playing directly into the Communist design of "confuse, divide, and conquer." As an American, I don't want a Democratic Administration "whitewash" or "cover-up" any more than I want a Republican smear or witch hunt. As an American, I condemn a Republican "Fascist" just as much I condemn a Democratic "Communist." I condemn a Democrat "Fascist" just as much as I condemn a Republican "Communist." They are equally dangerous to you and me and to our country. As an American, I want to see our nation recapture the strength and unity it once had when we fought the enemy instead of ourselves. It is with these thoughts that I have drafted what I call a "Declaration of Conscience." I am gratified that Senator Tobey, Senator Aiken, Senator Morse, Senator Ives, Senator Thye, and Senator Hendrickson have concurred in that declaration and have authorized me to announce their concurrence.`, + expected: 'very complex', + acceptable: ['moderately complex', 'exceedingly complex'], + }, +]; + +describeIntegration.concurrent('Vocabulary Evaluator - Comprehensive Test Suite', () => { + let evaluator: VocabularyEvaluator; + + beforeAll(() => { + if (SKIP_INTEGRATION) { + console.log('⏭️ Skipping integration tests (no API keys or RUN_INTEGRATION_TESTS not set)'); + return; + } + + evaluator = new VocabularyEvaluator({ + googleApiKey: process.env.GOOGLE_API_KEY!, + openaiApiKey: process.env.OPENAI_API_KEY!, + }); + + console.log('\n' + '='.repeat(80)); + console.log('VOCABULARY EVALUATOR - TEST SUITE (PARALLEL)'); + console.log('='.repeat(80)); + console.log(`Running ${TEST_CASES.length} test cases with up to 3 attempts each`); + console.log('Short-circuiting on first expected match'); + console.log('Checking acceptable values if no expected match'); + console.log('='.repeat(80)); + }); + + // Generate individual test for each case + TEST_CASES.forEach((testCase) => { + it.concurrent(`${testCase.id}: Grade ${testCase.grade} - ${testCase.expected}`, async () => { + // Buffer all logs to print atomically at the end (prevents interleaving in parallel tests) + const logBuffer: string[] = []; + + // Test header + logBuffer.push('\n' + '='.repeat(80)); + logBuffer.push(`Test Case ${testCase.id} | Grade: ${testCase.grade}`); + logBuffer.push('='.repeat(80)); + logBuffer.push(`Expected Complexity: ${testCase.expected}`); + logBuffer.push(`Text Preview: ${testCase.text.substring(0, 100)}...`); + logBuffer.push(''); + + // Run the evaluation (returns logs instead of printing) + const maxAttempts = 3; + const result = await runEvaluatorTest(testCase, { + evaluator, + extractResult: (r) => r.score, + maxAttempts, + }); + + // Add evaluation logs to buffer (includes detailed summary) + logBuffer.push(...result.logs); + + // Print all logs atomically at the end - single console.log to prevent interleaving + console.log(logBuffer.join('\n')); + + // Assert that we got a match within maxAttempts (expected or acceptable) + expect(result.matched).toBe(true); + expect(result.matchedOnAttempt).toBeDefined(); + expect(result.matchedOnAttempt).toBeLessThanOrEqual(maxAttempts); + }, TEST_TIMEOUT_MS); + }); +}); diff --git a/sdks/typescript/tests/unit/evaluators/validation.test.ts b/sdks/typescript/tests/unit/evaluators/validation.test.ts new file mode 100644 index 0000000..74c095a --- /dev/null +++ b/sdks/typescript/tests/unit/evaluators/validation.test.ts @@ -0,0 +1,142 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { VocabularyEvaluator } from '../../../src/evaluators/vocabulary.js'; +import { VALIDATION_LIMITS } from '../../../src/evaluators/base.js'; +import { ConfigurationError } from '../../../src/errors.js'; +import type { LLMProvider } from '../../../src/providers/base.js'; + +/** + * Comprehensive validation tests for input validation + * + * Tests the base evaluator validation logic that all evaluators inherit. + * Uses VocabularyEvaluator as the test subject since it extends BaseEvaluator. + * + * All tests use mocked providers to avoid real API calls. + */ + +// Mock providers +const createMockProvider = (): LLMProvider => ({ + generateStructured: vi.fn(), + generateText: vi.fn(), +}); + +// Mock the createProvider factory +vi.mock('../../../src/providers/index.js', () => ({ + createProvider: vi.fn(() => createMockProvider()), +})); + +// Mock telemetry to avoid real HTTP calls +vi.mock('../../../src/telemetry/client.js', () => { + return { + TelemetryClient: class MockTelemetryClient { + send = vi.fn().mockResolvedValue(undefined); + }, + }; +}); + +describe('Configuration Validation', () => { + it('should throw ConfigurationError when googleApiKey is missing', () => { + expect(() => new VocabularyEvaluator({ + googleApiKey: '', + openaiApiKey: 'test-openai-key', + })).toThrow(ConfigurationError); + }); + + it('should throw ConfigurationError when openaiApiKey is missing', () => { + expect(() => new VocabularyEvaluator({ + googleApiKey: 'test-google-key', + openaiApiKey: '', + })).toThrow(ConfigurationError); + }); +}); + +describe('Input Validation - Text Validation', () => { + let evaluator: VocabularyEvaluator; + + beforeEach(() => { + vi.clearAllMocks(); + + evaluator = new VocabularyEvaluator({ + googleApiKey: 'test-google-key', + openaiApiKey: 'test-openai-key', + telemetry: false, + }); + }); + + describe('Empty text validation', () => { + it.each([ + ['empty string', ''], + ['spaces only', ' '], + ['tabs only', '\t\t\t'], + ['newlines only', '\n\n\n'], + ['mixed whitespace', ' \t\n '], + ])('should reject %s', async (_label, text) => { + await expect(evaluator.evaluate(text, '5')) + .rejects.toThrow('Text cannot be empty or contain only whitespace'); + }); + }); + + describe('Minimum length validation', () => { + it(`should reject text shorter than ${VALIDATION_LIMITS.MIN_TEXT_LENGTH} characters`, async () => { + const shortText = 'Hello wo'; // 8 chars after trim + await expect(evaluator.evaluate(shortText, '5')) + .rejects.toThrow(`Text is too short. Minimum length is ${VALIDATION_LIMITS.MIN_TEXT_LENGTH} characters, received 8 characters`); + }); + }); + + describe('Maximum length validation', () => { + it(`should reject text longer than ${VALIDATION_LIMITS.MAX_TEXT_LENGTH.toLocaleString()} characters`, async () => { + const longText = 'a'.repeat(VALIDATION_LIMITS.MAX_TEXT_LENGTH + 1); + + await expect(evaluator.evaluate(longText, '5')) + .rejects.toThrow(new RegExp(`Text is too long\\. Maximum length is ${VALIDATION_LIMITS.MAX_TEXT_LENGTH.toLocaleString()} characters, received ${(VALIDATION_LIMITS.MAX_TEXT_LENGTH + 1).toLocaleString()} characters`)); + }); + }); +}); + +describe('Input Validation - Grade Validation', () => { + let evaluator: VocabularyEvaluator; + + beforeEach(() => { + vi.clearAllMocks(); + + evaluator = new VocabularyEvaluator({ + googleApiKey: 'test-google-key', + openaiApiKey: 'test-openai-key', + telemetry: false, + }); + }); + + describe('Valid grade range', () => { + it.each([ + ['K', 'K'], + ['1', '1'], + ['2', '2'], + ])('should reject grade %s (below minimum)', async (_label, grade) => { + const validText = 'This is a sample text for testing.'; + + await expect(evaluator.evaluate(validText, grade)) + .rejects.toThrow(`Invalid grade "${grade}". Supported grades for this evaluator: 3, 4, 5, 6, 7, 8, 9, 10, 11, 12`); + }); + + it.each([ + ['13', '13'], + ['99', '99'], + ])('should reject grade %s (above maximum)', async (_label, grade) => { + const validText = 'This is a sample text for testing.'; + + await expect(evaluator.evaluate(validText, grade)) + .rejects.toThrow(`Invalid grade "${grade}". Supported grades for this evaluator: 3, 4, 5, 6, 7, 8, 9, 10, 11, 12`); + }); + + it.each([ + ['invalid', 'invalid'], + ['grade5', 'grade5'], + ['empty string', ''], + ])('should reject grade %s (invalid format)', async (_label, grade) => { + const validText = 'This is a sample text for testing.'; + + await expect(evaluator.evaluate(validText, grade)) + .rejects.toThrow(`Invalid grade "${grade}". Supported grades for this evaluator: 3, 4, 5, 6, 7, 8, 9, 10, 11, 12`); + }); + }); +}); diff --git a/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts b/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts new file mode 100644 index 0000000..2ce906a --- /dev/null +++ b/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts @@ -0,0 +1,250 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { VocabularyEvaluator } from '../../../src/evaluators/vocabulary.js'; +import type { LLMProvider } from '../../../src/providers/base.js'; + +/** + * Comprehensive unit tests for VocabularyEvaluator + * + * These tests verify: + * - Constructor validation + * - Successful evaluation flow (both stages) + * - Error handling (LLM failures, validation errors) + * - Telemetry behavior (success/error cases) + * - Token usage aggregation + * - Edge cases + */ + +// Mock providers +const createMockProvider = (): LLMProvider => ({ + generateStructured: vi.fn(), + generateText: vi.fn(), +}); + +// Mock the createProvider factory +vi.mock('../../../src/providers/index.js', () => ({ + createProvider: vi.fn(() => createMockProvider()), +})); + +// Mock telemetry to avoid real HTTP calls +vi.mock('../../../src/telemetry/client.js', () => { + return { + TelemetryClient: class MockTelemetryClient { + send = vi.fn().mockResolvedValue(undefined); + }, + }; +}); + +describe('VocabularyEvaluator - Constructor Validation', () => { + it('should throw error when Google API key is missing', () => { + expect(() => new VocabularyEvaluator({ + googleApiKey: '', + openaiApiKey: 'test-openai-key', + })).toThrow('Google API key is required. Pass googleApiKey in config.'); + }); + + it('should throw error when OpenAI API key is missing', () => { + expect(() => new VocabularyEvaluator({ + googleApiKey: 'test-google-key', + openaiApiKey: '', + })).toThrow('OpenAI API key is required. Pass openaiApiKey in config.'); + }); + +}); + +describe('VocabularyEvaluator - Evaluation Flow', () => { + let evaluator: VocabularyEvaluator; + let mockBackgroundProvider: LLMProvider; + let mockComplexityProvider: LLMProvider; + + beforeEach(() => { + vi.clearAllMocks(); + + // Create evaluator (providers will be mocked) + evaluator = new VocabularyEvaluator({ + googleApiKey: 'test-google-key', + openaiApiKey: 'test-openai-key', + telemetry: false, // Disable telemetry for most tests + }); + + // Get references to the mocked providers + // @ts-expect-error Accessing private property for testing + mockBackgroundProvider = evaluator.backgroundKnowledgeProvider; + // @ts-expect-error Accessing private property for testing + // Tests use grade 5+, which routes to otherGradesComplexityProvider (GPT-4.1) + mockComplexityProvider = evaluator.otherGradesComplexityProvider; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('Successful Evaluation Flow', () => { + it('should successfully evaluate text through both stages', async () => { + const testText = 'The mitochondria is the powerhouse of the cell.'; + const testGrade = '5'; + + // Mock background knowledge response + vi.mocked(mockBackgroundProvider.generateText).mockResolvedValue({ + text: 'Students at grade 5 typically understand basic cell biology concepts.', + usage: { + inputTokens: 100, + outputTokens: 50, + }, + latencyMs: 500, + }); + + // Mock complexity evaluation response + vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ + data: { + complexity_score: 'moderately complex', + reasoning: 'The text uses grade-appropriate vocabulary.', + factors: ['Academic terminology', 'Clear structure'], + }, + model: 'gemini-2.5-pro', + usage: { + inputTokens: 200, + outputTokens: 100, + }, + latencyMs: 800, + }); + + // Execute evaluation + const result = await evaluator.evaluate(testText, testGrade); + + // Verify result structure + expect(result.score).toBe('moderately complex'); + expect(result.reasoning).toContain('grade-appropriate vocabulary'); + expect(result.metadata).toBeDefined(); + expect(result.metadata.model).toBe('openai:gpt-4o-2024-11-20 + openai:gpt-4.1-2025-04-14'); + expect(result.metadata.processingTimeMs).toBeGreaterThan(0); + + // Verify both providers were called + expect(mockBackgroundProvider.generateText).toHaveBeenCalledTimes(1); + expect(mockComplexityProvider.generateStructured).toHaveBeenCalledTimes(1); + + // Verify background knowledge call + const bgCall = vi.mocked(mockBackgroundProvider.generateText).mock.calls[0]; + expect(bgCall[0][0].content).toContain(testText); + expect(bgCall[1]).toBe(0); // temperature = 0 + + // Verify complexity call includes background knowledge + const complexityCall = vi.mocked(mockComplexityProvider.generateStructured).mock.calls[0]; + expect(complexityCall[0].messages[1].content).toContain(testText); + expect(complexityCall[0].schema).toBeDefined(); + expect(complexityCall[0].temperature).toBe(0); + }); + +}); + + describe('Error Handling', () => { + it('should handle background knowledge API failure', async () => { + const testText = 'Test text here for API failure'; + const testGrade = '5'; + + // Mock background knowledge failure + vi.mocked(mockBackgroundProvider.generateText).mockRejectedValue( + new Error('API timeout') + ); + + // Should propagate the error + await expect(evaluator.evaluate(testText, testGrade)) + .rejects.toThrow('API timeout'); + + // Verify complexity provider was never called + expect(mockComplexityProvider.generateStructured).not.toHaveBeenCalled(); + }); + + it('should handle complexity evaluation API failure', async () => { + const testText = 'Test text here for complexity failure'; + const testGrade = '6'; + + // Mock successful background knowledge + vi.mocked(mockBackgroundProvider.generateText).mockResolvedValue({ + text: 'Background knowledge', + usage: { inputTokens: 100, outputTokens: 50 }, + latencyMs: 500, + }); + + // Mock complexity evaluation failure + vi.mocked(mockComplexityProvider.generateStructured).mockRejectedValue( + new Error('Schema validation failed') + ); + + // Should propagate the error + await expect(evaluator.evaluate(testText, testGrade)) + .rejects.toThrow('Schema validation failed'); + + // Verify background provider was called (stage 1 completed) + expect(mockBackgroundProvider.generateText).toHaveBeenCalledTimes(1); + }); + + }); + + describe('Response Structure', () => { + it('should return correct result structure', async () => { + vi.mocked(mockBackgroundProvider.generateText).mockResolvedValue({ + text: 'Background knowledge', + usage: { inputTokens: 100, outputTokens: 50 }, + latencyMs: 500, + }); + + vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ + data: { + complexity_score: 'moderately complex', + reasoning: 'Detailed reasoning here', + factors: ['Factor 1', 'Factor 2'], + }, + model: 'gemini-2.5-pro', + usage: { inputTokens: 200, outputTokens: 100 }, + latencyMs: 800, + }); + + const result = await evaluator.evaluate('Test text here', '5'); + + // Verify result structure + expect(result).toHaveProperty('score'); + expect(result).toHaveProperty('reasoning'); + expect(result).toHaveProperty('metadata'); + expect(result).toHaveProperty('_internal'); + + // Verify metadata structure + expect(result.metadata).toHaveProperty('promptVersion'); + expect(result.metadata).toHaveProperty('model'); + expect(result.metadata).toHaveProperty('timestamp'); + expect(result.metadata).toHaveProperty('processingTimeMs'); + + // Verify metadata values + expect(result.metadata.promptVersion).toBe('1.2.0'); + expect(result.metadata.model).toBe('openai:gpt-4o-2024-11-20 + openai:gpt-4.1-2025-04-14'); + expect(result.metadata.timestamp).toBeInstanceOf(Date); + expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); // Mocked calls can be instant (0ms) + }); + + it('should include internal data', async () => { + vi.mocked(mockBackgroundProvider.generateText).mockResolvedValue({ + text: 'Background knowledge', + usage: { inputTokens: 100, outputTokens: 50 }, + latencyMs: 500, + }); + + const mockComplexityData = { + complexity_score: 'moderately complex', + reasoning: 'Detailed reasoning', + factors: ['Factor 1', 'Factor 2'], + analysis: 'Deep analysis', + }; + + vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ + data: mockComplexityData, + model: 'gemini-2.5-pro', + usage: { inputTokens: 200, outputTokens: 100 }, + latencyMs: 800, + }); + + const result = await evaluator.evaluate('Test text here', '5'); + + // Verify internal data is included + expect(result._internal).toEqual(mockComplexityData); + }); + }); +}); diff --git a/sdks/typescript/tests/unit/features/readability.test.ts b/sdks/typescript/tests/unit/features/readability.test.ts new file mode 100644 index 0000000..2c1eda6 --- /dev/null +++ b/sdks/typescript/tests/unit/features/readability.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect } from 'vitest'; +import { calculateFleschKincaidGrade } from '../../../src/features/readability.js'; + +describe('calculateFleschKincaidGrade', () => { + it('should calculate FK grade for simple text', () => { + const text = 'The cat sat on the mat. The dog ran away.'; + const grade = calculateFleschKincaidGrade(text); + + expect(grade).toBeLessThan(5); + expect(typeof grade).toBe('number'); + }); + + it('should handle empty text', () => { + const grade = calculateFleschKincaidGrade(''); + expect(grade).toBe(0); + }); + + it('should calculate higher grade for complex text', () => { + const simpleText = 'The cat sat.'; + const complexText = 'The mitochondria, known as the powerhouse of cellular respiration, facilitates biochemical processes.'; + + const simpleGrade = calculateFleschKincaidGrade(simpleText); + const complexGrade = calculateFleschKincaidGrade(complexText); + + expect(complexGrade).toBeGreaterThan(simpleGrade); + }); +}); diff --git a/sdks/typescript/tests/unit/telemetry/utils.test.ts b/sdks/typescript/tests/unit/telemetry/utils.test.ts new file mode 100644 index 0000000..77e50d2 --- /dev/null +++ b/sdks/typescript/tests/unit/telemetry/utils.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { getSDKVersion } from '../../../src/telemetry/utils.js'; + +// UUID v4 pattern +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +describe('Telemetry Utils', () => { + describe('generateClientId', () => { + // Reset module cache between tests so cachedClientId doesn't leak across tests + beforeEach(() => { + vi.resetModules(); + }); + + it('should generate a new UUID, create the config directory, and persist it when no config file exists', async () => { + const writeFileSync = vi.fn(); + const mkdirSync = vi.fn(); + vi.doMock('node:fs', () => ({ + readFileSync: vi.fn(() => { throw new Error('ENOENT'); }), + writeFileSync, + mkdirSync, + })); + vi.doMock('node:os', () => ({ homedir: vi.fn(() => '/home/user') })); + + const { generateClientId } = await import('../../../src/telemetry/utils.js'); + const id = generateClientId(); + + expect(id).toMatch(UUID_REGEX); + expect(mkdirSync).toHaveBeenCalledWith(expect.any(String), { recursive: true }); + expect(writeFileSync).toHaveBeenCalledOnce(); + const written = JSON.parse(writeFileSync.mock.calls[0][1] as string) as { + telemetry: { clientId: string }; + }; + expect(written.telemetry.clientId).toBe(id); + }); + + it('should not re-read from disk on repeated calls', async () => { + const readFileSync = vi.fn(() => { throw new Error('ENOENT'); }); + vi.doMock('node:fs', () => ({ + readFileSync, + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + })); + vi.doMock('node:os', () => ({ homedir: vi.fn(() => '/home/user') })); + + const { generateClientId } = await import('../../../src/telemetry/utils.js'); + + generateClientId(); + generateClientId(); + + expect(readFileSync).toHaveBeenCalledOnce(); + }); + + it('should read and return an existing client ID from config file without writing to disk', async () => { + const existingId = 'a1b2c3d4-e5f6-4789-ab01-cd23ef456789'; + const writeFileSync = vi.fn(); + const mkdirSync = vi.fn(); + vi.doMock('node:fs', () => ({ + readFileSync: vi.fn(() => JSON.stringify({ telemetry: { clientId: existingId } })), + writeFileSync, + mkdirSync, + })); + vi.doMock('node:os', () => ({ homedir: vi.fn(() => '/home/user') })); + + const { generateClientId } = await import('../../../src/telemetry/utils.js'); + + expect(generateClientId()).toBe(existingId); + expect(mkdirSync).not.toHaveBeenCalled(); + expect(writeFileSync).not.toHaveBeenCalled(); + }); + + it('should generate and persist a new UUID if config file exists but clientId is missing', async () => { + const writeFileSync = vi.fn(); + vi.doMock('node:fs', () => ({ + readFileSync: vi.fn(() => JSON.stringify({ telemetry: {} })), + writeFileSync, + mkdirSync: vi.fn(), + })); + vi.doMock('node:os', () => ({ homedir: vi.fn(() => '/home/user') })); + + const { generateClientId } = await import('../../../src/telemetry/utils.js'); + const id = generateClientId(); + + expect(id).toMatch(UUID_REGEX); + expect(writeFileSync).toHaveBeenCalledOnce(); + const written = JSON.parse(writeFileSync.mock.calls[0][1] as string) as { + telemetry: { clientId: string }; + }; + expect(written.telemetry.clientId).toBe(id); + }); + + it('should return a valid UUID without throwing when filesystem is read-only', async () => { + vi.doMock('node:fs', () => ({ + readFileSync: vi.fn(() => { throw new Error('ENOENT'); }), + writeFileSync: vi.fn(() => { throw new Error('EROFS'); }), + mkdirSync: vi.fn(() => { throw new Error('EROFS'); }), + })); + vi.doMock('node:os', () => ({ homedir: vi.fn(() => '/home/user') })); + + const { generateClientId } = await import('../../../src/telemetry/utils.js'); + + let id: string | undefined; + expect(() => { id = generateClientId(); }).not.toThrow(); + expect(id).toMatch(UUID_REGEX); + }); + }); + + describe('getSDKVersion', () => { + it('should return a valid version string', () => { + const version = getSDKVersion(); + + expect(version).toMatch(/^\d+\.\d+\.\d+$/); + }); + + it('should return same version on repeated calls (cached)', () => { + const version1 = getSDKVersion(); + const version2 = getSDKVersion(); + + expect(version1).toBe(version2); + }); + }); +}); diff --git a/sdks/typescript/tests/utils/index.ts b/sdks/typescript/tests/utils/index.ts new file mode 100644 index 0000000..e01a630 --- /dev/null +++ b/sdks/typescript/tests/utils/index.ts @@ -0,0 +1,18 @@ +/** + * Test utilities for evaluator testing + * + * @example + * ```typescript + * import { runTestWithRetry, runEvaluatorTest } from '../utils'; + * ``` + */ + +export { + runTestWithRetry, + runEvaluatorTest, + type TestAttempt, + type TestResult, + type RetryTestOptions, + type BaseTestCase, + type EvaluatorTestConfig, +} from './test-helpers.js'; diff --git a/sdks/typescript/tests/utils/test-helpers.ts b/sdks/typescript/tests/utils/test-helpers.ts new file mode 100644 index 0000000..91a6be7 --- /dev/null +++ b/sdks/typescript/tests/utils/test-helpers.ts @@ -0,0 +1,254 @@ +/** + * Streamlined test utilities for evaluator testing + */ + +export interface TestAttempt { + attempt: number; + result: T; + matched: boolean; +} + +export interface TestResult { + matched: boolean; + matchedOnAttempt?: number; + matchType?: 'expected' | 'acceptable'; // How the match occurred + totalAttempts: number; + attempts: TestAttempt[]; + allResults: T[]; + logs: string[]; // Buffered log messages for atomic printing +} + +export interface RetryTestOptions { + /** Function that executes the test and returns the actual output */ + testFn: (input: TInput) => Promise; + + /** Input to pass to the test function */ + input: TInput; + + /** Expected output value */ + expected: TOutput; + + /** Maximum number of attempts (default: 3) */ + maxAttempts?: number; + + /** Custom comparison function (default: strict equality) */ + compareFn?: (actual: TOutput, expected: TOutput) => boolean; + + /** Optional callback after each attempt */ + onAttempt?: (attempt: number, result: TOutput, matched: boolean) => void; +} + +/** + * Default comparison function (case-insensitive string comparison) + */ +function defaultCompareFn(actual: T, expected: T): boolean { + if (typeof actual === 'string' && typeof expected === 'string') { + return actual.toLowerCase() === expected.toLowerCase(); + } + return actual === expected; +} + +/** + * Runs a test function multiple times with retry logic and short-circuiting. + */ +export async function runTestWithRetry( + options: RetryTestOptions +): Promise> { + const { + testFn, + input, + expected, + maxAttempts = 3, + compareFn = defaultCompareFn, + onAttempt, + } = options; + + const attempts: TestAttempt[] = []; + let matched = false; + let matchedOnAttempt: number | undefined; + + for (let attemptNum = 1; attemptNum <= maxAttempts; attemptNum++) { + const result = await testFn(input); + const isMatch = compareFn(result, expected); + + attempts.push({ + attempt: attemptNum, + result, + matched: isMatch, + }); + + if (onAttempt) { + onAttempt(attemptNum, result, isMatch); + } + + // Short-circuit on match + if (isMatch) { + matched = true; + matchedOnAttempt = attemptNum; + break; + } + } + + return { + matched, + matchedOnAttempt, + totalAttempts: attempts.length, + attempts, + allResults: attempts.map(a => a.result), + logs: [], // No logs for this simple retry function + }; +} + +/** + * Generic test case structure + * All evaluator-specific test cases extend this + */ +export interface BaseTestCase { + id: string; + text: string; + grade?: string; // Optional: some evaluators need it, some don't + expected: string; // Expected output value (checked on each attempt) + acceptable?: string[]; // Acceptable adjacent values (checked if no expected match after all retries) +} + +/** + * Configuration for running evaluator tests + */ +export interface EvaluatorTestConfig { + /** The evaluator instance to test */ + evaluator: TEvaluator; + + /** Function to extract the result to compare from evaluation output */ + extractResult: (evalResult: any) => string; + + /** Maximum retry attempts (default: 3) */ + maxAttempts?: number; +} + +/** + * Generic evaluator test runner + * Works for any evaluator with retry logic + * + * @example + * ```typescript + * // Vocabulary evaluator + * const result = await runEvaluatorTest( + * { + * id: 'V1', + * text: 'Sample text...', + * grade: '3', + * expected: 'very complex' + * }, + * { + * evaluator: vocabularyEvaluator, + * extractResult: (r) => r.score + * } + * ); + * + * // Grade level evaluator + * const result = await runEvaluatorTest( + * { + * id: 'GLA1', + * text: 'Sample text...', + * expected: '6-8' + * }, + * { + * evaluator: gradeLevelEvaluator, + * extractResult: (r) => r.score.grade + * } + * ); + * ``` + */ +export async function runEvaluatorTest( + testCase: BaseTestCase, + config: EvaluatorTestConfig +): Promise> { + const { evaluator, extractResult, maxAttempts = 3 } = config; + const compareFn = defaultCompareFn; + + // Buffer logs to print atomically at the end (prevents interleaving in parallel tests) + const logBuffer: string[] = []; + + // Log test criteria upfront + logBuffer.push(`\n Expected: "${testCase.expected}"`); + if (testCase.acceptable && testCase.acceptable.length > 0) { + logBuffer.push(` Acceptable: [${testCase.acceptable.map(v => `"${v}"`).join(', ')}]`); + } + logBuffer.push(''); + + const attempts: TestAttempt[] = []; + let matched = false; + let matchedOnAttempt: number | undefined; + let matchType: 'expected' | 'acceptable' | undefined; + + // Phase 1: Try to match expected value (short-circuit on match) + for (let attemptNum = 1; attemptNum <= maxAttempts; attemptNum++) { + const result = testCase.grade + ? await evaluator.evaluate(testCase.text, testCase.grade) + : await evaluator.evaluate(testCase.text); + + const actualValue = extractResult(result); + const isExpectedMatch = compareFn(actualValue, testCase.expected); + + attempts.push({ + attempt: attemptNum, + result: actualValue, + matched: isExpectedMatch, + }); + + logBuffer.push(` Attempt ${attemptNum}: "${actualValue}" ${isExpectedMatch ? '✓ EXPECTED MATCH' : '✗'}`); + + // Short-circuit on expected match + if (isExpectedMatch) { + matched = true; + matchedOnAttempt = attemptNum; + matchType = 'expected'; + break; + } + } + + // Phase 2: If no expected match, check if any result is in acceptable range + // Only check acceptable values if they are defined and non-empty + if (!matched && testCase.acceptable?.length) { + logBuffer.push('\n No expected match. Checking acceptable values...'); + + for (let i = 0; i < attempts.length; i++) { + const attemptResult = attempts[i].result; + const isAcceptable = testCase.acceptable.some(acceptable => + compareFn(attemptResult, acceptable) + ); + + if (isAcceptable) { + matched = true; + matchedOnAttempt = i + 1; + matchType = 'acceptable'; + logBuffer.push(` ✓ ACCEPTABLE MATCH: Attempt ${matchedOnAttempt} result "${attemptResult}" is in acceptable range`); + break; + } + } + + if (!matched) { + logBuffer.push(` ✗ NO MATCH: None of the attempts matched expected or acceptable values`); + } + } + + // Summary logging + logBuffer.push('\n Summary:'); + logBuffer.push(` All Results: [${attempts.map(a => `"${a.result}"`).join(', ')}]`); + if (matched) { + logBuffer.push(` Status: ✓ PASS (matched ${matchType} on attempt ${matchedOnAttempt})`); + } else { + logBuffer.push(` Status: ✗ FAIL (no match after ${attempts.length} attempts)`); + } + + // Return logs for atomic printing by the caller + return { + matched, + matchedOnAttempt, + matchType, + totalAttempts: attempts.length, + attempts, + allResults: attempts.map(a => a.result), + logs: logBuffer, + }; +} \ No newline at end of file diff --git a/sdks/typescript/vitest.config.ts b/sdks/typescript/vitest.config.ts index 9eb9a49..06f43e3 100644 --- a/sdks/typescript/vitest.config.ts +++ b/sdks/typescript/vitest.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from 'vitest/config'; +import { loadEnv } from 'vite'; import { readFileSync } from 'fs'; import { resolve, dirname } from 'path'; import type { Plugin } from 'vite'; @@ -18,12 +19,13 @@ function txtPlugin(): Plugin { }; } -export default defineConfig({ +export default defineConfig(({ mode }) => ({ plugins: [txtPlugin()], test: { globals: true, environment: 'node', passWithNoTests: true, + env: loadEnv(mode, process.cwd(), ''), coverage: { provider: 'v8', reporter: ['text', 'json', 'html'], @@ -36,4 +38,4 @@ export default defineConfig({ ], }, }, -}); +})); From 5b253e30e969fbba366a6942d73d07b8c65fd837 Mon Sep 17 00:00:00 2001 From: Adnan Hussain <61515575+adnanrhussain@users.noreply.github.com> Date: Mon, 2 Mar 2026 22:41:22 -0800 Subject: [PATCH 4/9] feat: Implement SS evaluator (#13) * feat: Implement SS evaluator --- sdks/typescript/README.md | 49 ++- sdks/typescript/docs/telemetry.md | 6 +- sdks/typescript/src/evaluators/index.ts | 6 + .../src/evaluators/sentence-structure.ts | 384 ++++++++++++++++++ sdks/typescript/src/features/index.ts | 2 + .../src/features/sentence-features.ts | 228 +++++++++++ sdks/typescript/src/index.ts | 17 + .../prompts/sentence-structure/analysis.ts | 22 + .../prompts/sentence-structure/complexity.ts | 52 +++ .../src/prompts/sentence-structure/index.ts | 6 + .../src/schemas/sentence-structure.ts | 124 ++++++ .../sentence-structure.integration.test.ts | 128 ++++++ .../evaluators/sentence-structure.test.ts | 249 ++++++++++++ 13 files changed, 1269 insertions(+), 4 deletions(-) create mode 100644 sdks/typescript/src/evaluators/sentence-structure.ts create mode 100644 sdks/typescript/src/features/sentence-features.ts create mode 100644 sdks/typescript/src/prompts/sentence-structure/analysis.ts create mode 100644 sdks/typescript/src/prompts/sentence-structure/complexity.ts create mode 100644 sdks/typescript/src/prompts/sentence-structure/index.ts create mode 100644 sdks/typescript/src/schemas/sentence-structure.ts create mode 100644 sdks/typescript/tests/integration/sentence-structure.integration.test.ts create mode 100644 sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md index f4209df..1fc51cb 100644 --- a/sdks/typescript/README.md +++ b/sdks/typescript/README.md @@ -40,7 +40,7 @@ Evaluates vocabulary complexity using the Qual Text Complexity rubric (SAP). **Supported Grades:** 3-12 -**Uses:** Google Gemini 2.5 Pro + OpenAI GPT-4o +**Uses:** OpenAI GPT-4o (background knowledge) + Google Gemini 2.5 Pro (grades 3–4) / OpenAI GPT-4.1 (grades 5–12) **Constructor:** ```typescript @@ -74,6 +74,53 @@ await evaluator.evaluate(text: string, grade: string) } ``` +--- + +### 2. Sentence Structure Evaluator + +Evaluates sentence structure complexity based on grammatical features. + +**Supported Grades:** K-12 + +**Uses:** OpenAI GPT-4o + +**Constructor:** +```typescript +const evaluator = new SentenceStructureEvaluator({ + openaiApiKey: string; // Required - OpenAI API key + maxRetries?: number; // Optional - Max retry attempts (default: 2) + telemetry?: boolean | TelemetryOptions; // Optional (default: true) + logger?: Logger; // Optional - Custom logger + logLevel?: LogLevel; // Optional - Logging verbosity (default: WARN) +}); +``` + +**API:** +```typescript +await evaluator.evaluate(text: string, grade: string) +``` + +**Returns:** +```typescript +{ + score: 'Slightly Complex' | 'Moderately Complex' | 'Very Complex' | 'Exceedingly Complex'; + reasoning: string; + metadata: { + promptVersion: string; + model: string; + timestamp: Date; + processingTimeMs: number; + }; + _internal: { + sentenceAnalysis: SentenceAnalysis; + features: SentenceFeatures; + complexity: ComplexityClassification; + }; +} +``` + +--- + ## Error Handling The SDK provides specific error types to help you handle different scenarios: diff --git a/sdks/typescript/docs/telemetry.md b/sdks/typescript/docs/telemetry.md index 991479f..5a1ddc1 100644 --- a/sdks/typescript/docs/telemetry.md +++ b/sdks/typescript/docs/telemetry.md @@ -25,11 +25,11 @@ If you prefer not to send any telemetry, you can disable it entirely — see [Di "timestamp": "2026-02-05T19:30:00.000Z", "sdk_version": "0.1.0", "evaluator_type": "vocabulary", - "grade": "5", + "grade": "3", "status": "success", "latency_ms": 3500, "text_length_chars": 456, - "provider": "google:gemini-2.5-pro+openai:gpt-4o", + "provider": "openai:gpt-4o-2024-11-20 + google:gemini-2.5-pro", "token_usage": { "input_tokens": 650, "output_tokens": 350 @@ -72,7 +72,7 @@ If you prefer not to send any telemetry, you can disable it entirely — see [Di | `latency_ms` | Total evaluation time in milliseconds | | `text_length_chars` | Length of input text in characters | | `provider` | LLM provider(s) used (e.g., "openai:gpt-4o", "google:gemini-2.5-pro+openai:gpt-4o") | -| `token_usage` | Total tokens consumed (input, output, total) | +| `token_usage` | Total tokens consumed (input, output) | | `input_text` | The text being evaluated (only included if `recordInputs: true`) | | `metadata.stage_details` | Per-stage breakdown for multi-stage evaluators (optional) | diff --git a/sdks/typescript/src/evaluators/index.ts b/sdks/typescript/src/evaluators/index.ts index 2f6ff9d..e96898e 100644 --- a/sdks/typescript/src/evaluators/index.ts +++ b/sdks/typescript/src/evaluators/index.ts @@ -5,3 +5,9 @@ export { evaluateVocabulary, type VocabularyEvaluatorConfig, } from './vocabulary.js'; + +export { + SentenceStructureEvaluator, + evaluateSentenceStructure, + type SentenceStructureEvaluatorConfig, +} from './sentence-structure.js'; diff --git a/sdks/typescript/src/evaluators/sentence-structure.ts b/sdks/typescript/src/evaluators/sentence-structure.ts new file mode 100644 index 0000000..67597f4 --- /dev/null +++ b/sdks/typescript/src/evaluators/sentence-structure.ts @@ -0,0 +1,384 @@ +import type { LLMProvider } from '../providers/index.js'; +import { createProvider } from '../providers/index.js'; +import { + SentenceAnalysisSchema, + ComplexityClassificationSchema, + type SentenceAnalysis, + type SentenceFeatures, + type ComplexityClassification, +} from '../schemas/sentence-structure.js'; +import { calculateReadabilityMetrics, addEngineeredFeatures, featuresToJSON } from '../features/index.js'; +import { + getSystemPromptAnalysis, + getUserPromptAnalysis, + getSystemPromptComplexity, + getUserPromptComplexity, +} from '../prompts/sentence-structure/index.js'; +import type { EvaluationResult, ComplexityLevel } from '../schemas/index.js'; +import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; +import type { StageDetail } from '../telemetry/index.js'; +import { ConfigurationError, ValidationError, wrapProviderError } from '../errors.js'; + +/** + * Valid grade levels (K-12) + */ +const VALID_GRADES = new Set(['K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']); + +/** + * Internal data structure for sentence structure evaluation + */ +interface SentenceStructureInternal { + sentenceAnalysis: SentenceAnalysis; + features: SentenceFeatures; + complexity: ComplexityClassification; +} + +/** + * Normalize complexity label to handle LLM output variations + * Ported from Python normalize_label function + */ +function normalizeLabel(label: string | null | undefined): string | null { + if (!label) { + return null; + } + + const normalized = label.trim().toLowerCase(); + const mapping: Record = { + 'slightly complex': 'Slightly Complex', + 'moderately complex': 'Moderately Complex', + 'very complex': 'Very Complex', + 'exceedingly complex': 'Exceedingly Complex', + 'extremely complex': 'Exceedingly Complex', // Maps to Exceedingly Complex + }; + + return mapping[normalized] || null; // Return null if no mapping found +} + +/** + * Configuration for SentenceStructureEvaluator + */ +export interface SentenceStructureEvaluatorConfig extends BaseEvaluatorConfig { + /** OpenAI API key for sentence analysis and complexity evaluation (uses GPT-4o) */ + openaiApiKey: string; +} + +/** + * Sentence Structure Evaluator + * + * Evaluates sentence structure complexity of educational texts relative to grade level. + * Uses a 2-stage process: + * 1. Analyze grammatical structure (sentence types, clauses, phrases, etc.) + * 2. Classify complexity using features and grade-specific rubric + * + * Based on SCASS Text Complexity rubric with 4 levels: + * - Slightly Complex + * - Moderately Complex + * - Very Complex + * - Exceedingly Complex + * + * @example + * ```typescript + * const evaluator = new SentenceStructureEvaluator({ + * openaiApiKey: process.env.OPENAI_API_KEY + * }); + * + * const result = await evaluator.evaluate(text, "3"); + * console.log(result.score); // "Moderately Complex" + * console.log(result.reasoning); + * ``` + */ +export class SentenceStructureEvaluator extends BaseEvaluator { + private analysisProvider: LLMProvider; + private complexityProvider: LLMProvider; + + constructor(config: SentenceStructureEvaluatorConfig) { + // Call base constructor for common setup (telemetry, etc.) + super(config); + + // Validate required API keys + if (!config.openaiApiKey) { + throw new ConfigurationError('OpenAI API key is required. Pass openaiApiKey in config.'); + } + + // Create OpenAI GPT-4o provider for both stages + this.analysisProvider = createProvider({ + type: 'openai', + model: 'gpt-4o', + apiKey: config.openaiApiKey, + maxRetries: this.config.maxRetries, + }); + + this.complexityProvider = createProvider({ + type: 'openai', + model: 'gpt-4o', + apiKey: config.openaiApiKey, + maxRetries: this.config.maxRetries, + }); + } + + // Implement abstract methods from BaseEvaluator + protected getEvaluatorType(): string { + return 'sentence-structure'; + } + + /** + * Evaluate sentence structure complexity for a given text and grade level + * + * @param text - The text to evaluate + * @param grade - The target grade level (K-12) + * @returns Evaluation result with complexity score and detailed analysis + * @throws {ValidationError} If text is empty, too short/long, or grade is invalid + * @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError) + */ + async evaluate( + text: string, + grade: string + ): Promise> { + this.logger.info('Starting sentence structure evaluation', { + evaluator: 'sentence-structure', + operation: 'evaluate', + grade, + textLength: text.length, + }); + + const startTime = Date.now(); + const stageDetails: StageDetail[] = []; + + try { + // Validate inputs — inside try so validation errors are telemetered. + this.validateText(text); + this.validateGrade(grade, VALID_GRADES); + this.logger.debug('Stage 1: Analyzing sentence structure', { + evaluator: 'sentence-structure', + operation: 'sentence_analysis', + }); + // Stage 1: Analyze sentence structure + const analysisResponse = await this.analyzeSentenceStructure(text); + + stageDetails.push({ + stage: 'sentence_analysis', + provider: 'openai:gpt-4o', + latency_ms: analysisResponse.latencyMs, + token_usage: { + input_tokens: analysisResponse.usage.inputTokens, + output_tokens: analysisResponse.usage.outputTokens, + }, + }); + + // Compute engineered features + const features = addEngineeredFeatures(analysisResponse.data); + + this.logger.debug('Stage 2: Classifying complexity', { + evaluator: 'sentence-structure', + operation: 'complexity_classification', + }); + // Stage 2: Classify complexity + const complexityResponse = await this.classifyComplexity(features, grade, text); + + stageDetails.push({ + stage: 'complexity_classification', + provider: 'openai:gpt-4o', + latency_ms: complexityResponse.latencyMs, + token_usage: { + input_tokens: complexityResponse.usage.inputTokens, + output_tokens: complexityResponse.usage.outputTokens, + }, + }); + + const latencyMs = Date.now() - startTime; + + // Aggregate token usage + const totalTokenUsage = { + input_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.input_tokens || 0), 0), + output_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.output_tokens || 0), 0), + }; + + const result = { + score: complexityResponse.data.answer, + reasoning: complexityResponse.data.reasoning, + metadata: { + promptVersion: '1.2.0', + model: 'openai:gpt-4o', + timestamp: new Date(), + processingTimeMs: latencyMs, + }, + _internal: { + sentenceAnalysis: analysisResponse.data, + features, + complexity: complexityResponse.data, + }, + }; + + // Send success telemetry (fire-and-forget) + this.sendTelemetry({ + status: 'success', + latencyMs, + textLength: text.length, + grade, + provider: 'openai:gpt-4o', + tokenUsage: totalTokenUsage, + metadata: { + stage_details: stageDetails, + }, + inputText: text, + }).catch(() => { + // Ignore telemetry errors + }); + + this.logger.info('Sentence structure evaluation completed successfully', { + evaluator: 'sentence-structure', + operation: 'evaluate', + grade, + score: result.score, + processingTimeMs: latencyMs, + }); + + return result; + } catch (error) { + const latencyMs = Date.now() - startTime; + + // Log the error + this.logger.error('Sentence structure evaluation failed', { + evaluator: 'sentence-structure', + operation: 'evaluate', + grade, + error: error instanceof Error ? error : undefined, + processingTimeMs: latencyMs, + completedStages: stageDetails.length, + }); + + // Aggregate metrics from completed stages + const totalTokenUsage = stageDetails.length > 0 ? { + input_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.input_tokens || 0), 0), + output_tokens: stageDetails.reduce((sum, s) => sum + (s.token_usage?.output_tokens || 0), 0), + } : undefined; + + // Send failure telemetry (fire-and-forget) + this.sendTelemetry({ + status: 'error', + latencyMs, + textLength: text.length, + grade, + provider: 'openai:gpt-4o', + tokenUsage: totalTokenUsage, + errorCode: error instanceof Error ? error.name : 'UnknownError', + metadata: stageDetails.length > 0 ? { stage_details: stageDetails } : undefined, + inputText: text, + }).catch(() => { + // Ignore telemetry errors + }); + + // Re-throw validation errors as-is + if (error instanceof ValidationError) { + throw error; + } + + // Wrap provider errors into appropriate error types + throw wrapProviderError(error, 'Sentence structure evaluation failed'); + } + } + + /** + * Stage 1: Analyze sentence grammatical structure + * + * Analyzes sentence types, clauses, phrases, transitions, and other grammatical features + */ + private async analyzeSentenceStructure( + text: string + ): Promise<{ data: SentenceAnalysis; usage: { inputTokens: number; outputTokens: number }; latencyMs: number }> { + // Compute ground truth counts + const metrics = calculateReadabilityMetrics(text); + + const gtCountsStr = [ + `num_sentences: ${metrics.sentenceCount}`, + `num_words: ${metrics.wordCount}`, + `num_char: ${metrics.characterCount}`, + `num_syllable: ${metrics.syllableCount}`, + `flesch_kincaid_grade: ${metrics.fleschKincaidGrade}`, + ].join('\n'); + + const userPrompt = getUserPromptAnalysis(text, gtCountsStr); + + const response = await this.analysisProvider.generateStructured({ + messages: [ + { role: 'system', content: getSystemPromptAnalysis() }, + { role: 'user', content: userPrompt }, + ], + schema: SentenceAnalysisSchema, + temperature: 0, + }); + + return { + data: response.data, + usage: response.usage, + latencyMs: response.latencyMs, + }; + } + + /** + * Stage 2: Classify sentence structure complexity + * + * Uses engineered features and grade-specific rubric to classify complexity level + */ + private async classifyComplexity( + features: SentenceFeatures, + grade: string, + excerpt: string + ): Promise<{ data: ComplexityClassification; usage: { inputTokens: number; outputTokens: number }; latencyMs: number }> { + // Convert features to JSON string (cast to int by default, matching Python) + const featuresJSON = featuresToJSON(features, 1, true); + + const userPrompt = getUserPromptComplexity(featuresJSON, grade, excerpt); + + const response = await this.complexityProvider.generateStructured({ + messages: [ + { role: 'system', content: getSystemPromptComplexity() }, + { role: 'user', content: userPrompt }, + ], + schema: ComplexityClassificationSchema, + temperature: 0, + }); + + // Normalize label to handle LLM output variations + const normalizedAnswer = normalizeLabel(response.data.answer); + + if (!normalizedAnswer) { + throw new Error( + `Failed to normalize complexity label. Received unexpected value: "${response.data.answer}". ` + + `Expected one of: Slightly Complex, Moderately Complex, Very Complex, Exceedingly Complex, Extremely Complex.` + ); + } + + return { + data: { + ...response.data, + answer: normalizedAnswer as ComplexityLevel, + }, + usage: response.usage, + latencyMs: response.latencyMs, + }; + } +} + +/** + * Functional API for sentence structure evaluation + * + * @example + * ```typescript + * const result = await evaluateSentenceStructure( + * "The cat sat on the mat. It was sleeping peacefully.", + * "3", + * { + * openaiApiKey: process.env.OPENAI_API_KEY + * } + * ); + * ``` + */ +export async function evaluateSentenceStructure( + text: string, + grade: string, + config: SentenceStructureEvaluatorConfig +): Promise> { + const evaluator = new SentenceStructureEvaluator(config); + return evaluator.evaluate(text, grade); +} diff --git a/sdks/typescript/src/features/index.ts b/sdks/typescript/src/features/index.ts index 354830e..54a802e 100644 --- a/sdks/typescript/src/features/index.ts +++ b/sdks/typescript/src/features/index.ts @@ -3,3 +3,5 @@ export { calculateReadabilityMetrics, type ReadabilityMetrics, } from './readability.js'; + +export { addEngineeredFeatures, featuresToJSON, FEATURE_COLS } from './sentence-features.js'; diff --git a/sdks/typescript/src/features/sentence-features.ts b/sdks/typescript/src/features/sentence-features.ts new file mode 100644 index 0000000..69375ca --- /dev/null +++ b/sdks/typescript/src/features/sentence-features.ts @@ -0,0 +1,228 @@ +import type { SentenceAnalysis, SentenceFeatures } from '../schemas/sentence-structure.js'; + +/** + * Safe division helper (avoids division by zero) + */ +function safeDivision(numerator: number, denominator: number): number { + return denominator === 0 ? 0 : numerator / denominator; +} + +/** + * Calculate standard deviation of an array of numbers + */ +function standardDeviation(values: number[]): number { + if (values.length <= 1) return 0; + + const mean = values.reduce((sum, val) => sum + val, 0) / values.length; + const squaredDiffs = values.map((val) => Math.pow(val - mean, 2)); + const variance = squaredDiffs.reduce((sum, val) => sum + val, 0) / values.length; + + return Math.sqrt(variance); +} + +/** + * Categorize sentence lengths into short/medium/long/very long + */ +function categorizeSentenceLengths(wordCounts: number[]) { + if (!wordCounts || wordCounts.length === 0) { + return { + percent_short_sentences: 0, + percent_medium_sentences: 0, + percent_long_sentences: 0, + percent_very_long_sentences: 0, + }; + } + + let short = 0, + medium = 0, + long = 0, + veryLong = 0; + + for (const count of wordCounts) { + if (count <= 10) short++; + else if (count <= 20) medium++; + else if (count <= 30) long++; + else veryLong++; + } + + const total = wordCounts.length; + + return { + percent_short_sentences: (short / total) * 100, + percent_medium_sentences: (medium / total) * 100, + percent_long_sentences: (long / total) * 100, + percent_very_long_sentences: (veryLong / total) * 100, + }; +} + +/** + * Add engineered features to sentence analysis output + * Ported from Python add_engineered_features function + */ +export function addEngineeredFeatures(analysis: SentenceAnalysis): SentenceFeatures { + const numSentences = analysis.num_sentences; + const numWords = analysis.num_words; + + // Foundational Metrics + const avg_words_per_sentence = safeDivision(numWords, numSentences); + const sentence_length_variation = standardDeviation(analysis.sentence_word_counts); + const lengthCategories = categorizeSentenceLengths(analysis.sentence_word_counts); + + // Sentence Structure Percentages + const percent_simple_sentences = safeDivision(analysis.num_simple_sentences, numSentences) * 100; + const percent_compound_sentences = + safeDivision(analysis.num_compound_sentences, numSentences) * 100; + const percent_complex_sentences = safeDivision(analysis.num_complex_sentences, numSentences) * 100; + const percent_compound_complex_sentences = + safeDivision(analysis.num_compound_complex_sentences, numSentences) * 100; + const percent_other_sentences = safeDivision(analysis.num_other_sentences, numSentences) * 100; + + // Word Distribution Percentages + const percent_words_in_simple_sentences = + safeDivision(analysis.words_in_simple_sentences, numWords) * 100; + const percent_words_in_compound_sentences = + safeDivision(analysis.words_in_compound_sentences, numWords) * 100; + const percent_words_in_complex_sentences = + safeDivision(analysis.words_in_complex_sentences, numWords) * 100; + const percent_words_in_compound_complex_sentences = + safeDivision(analysis.words_in_compound_complex_sentences, numWords) * 100; + const percent_words_in_other_sentences = + safeDivision(analysis.words_in_other_sentences, numWords) * 100; + + // Subordination and Clausal Complexity + const avg_subordinates_per_sentence = safeDivision(analysis.num_subordinate_clauses, numSentences); + const avg_clauses_per_sentence = safeDivision(analysis.num_total_clauses, numSentences); + const percent_sentences_with_subordinate = + safeDivision(analysis.num_sentences_with_subordinate, numSentences) * 100; + const percent_sentences_with_multiple_subordinates = + safeDivision(analysis.num_sentences_with_multiple_subordinates, numSentences) * 100; + const percent_sentences_with_embedded_clauses = + safeDivision(analysis.num_sentences_with_embedded_clauses, numSentences) * 100; + + // Phrase Density (per 100 words) + const prep_phrase_density = safeDivision(analysis.num_prepositional_phrases, numWords) * 100; + const participle_phrase_density = safeDivision(analysis.num_participle_phrases, numWords) * 100; + const appositive_phrase_density = safeDivision(analysis.num_appositive_phrases, numWords) * 100; + + // Cohesion and Transitions + const total_transitions = analysis.num_simple_transitions + analysis.num_sophisticated_transitions; + const avg_transitions_per_sentence = safeDivision(total_transitions, numSentences); + const percent_sophisticated_transitions = + safeDivision(analysis.num_sophisticated_transitions, total_transitions) * 100; + + // Conceptual & Other + const percent_sentences_w_one_concept = + safeDivision(analysis.num_one_concept_sentences, numSentences) * 100; + const percent_sentences_w_multi_concept = + safeDivision(analysis.num_multi_concept_sentences, numSentences) * 100; + const percent_cleft_sentences = safeDivision(analysis.num_cleft_sentences, numSentences) * 100; + + return { + ...analysis, + avg_words_per_sentence, + sentence_length_variation, + ...lengthCategories, + percent_simple_sentences, + percent_compound_sentences, + percent_complex_sentences, + percent_compound_complex_sentences, + percent_other_sentences, + percent_words_in_simple_sentences, + percent_words_in_compound_sentences, + percent_words_in_complex_sentences, + percent_words_in_compound_complex_sentences, + percent_words_in_other_sentences, + avg_subordinates_per_sentence, + avg_clauses_per_sentence, + percent_sentences_with_subordinate, + percent_sentences_with_multiple_subordinates, + percent_sentences_with_embedded_clauses, + prep_phrase_density, + participle_phrase_density, + appositive_phrase_density, + avg_transitions_per_sentence, + percent_sophisticated_transitions, + percent_sentences_w_one_concept, + percent_sentences_w_multi_concept, + percent_cleft_sentences, + }; +} + +/** + * Feature columns used for complexity classification + * Must match the order and names from Python FEATURE_COLS + */ +export const FEATURE_COLS = [ + // Foundational & Distributional + 'avg_words_per_sentence', + 'sentence_length_variation', + 'percent_short_sentences', + 'percent_medium_sentences', + 'percent_long_sentences', + 'percent_very_long_sentences', + 'flesch_kincaid_grade', + // Sentence Structure (Grammatical Type) + 'percent_simple_sentences', + 'percent_compound_sentences', + 'percent_complex_sentences', + 'percent_compound_complex_sentences', + 'percent_other_sentences', + // Word Distribution + 'percent_words_in_simple_sentences', + 'percent_words_in_complex_sentences', + 'percent_words_in_compound_sentences', + 'percent_words_in_compound_complex_sentences', + 'percent_words_in_other_sentences', + // Clausal & Subordination + 'avg_subordinates_per_sentence', + 'avg_clauses_per_sentence', + 'percent_sentences_with_subordinate', + 'percent_sentences_with_multiple_subordinates', + 'percent_sentences_with_embedded_clauses', + // Phrase Density + 'prep_phrase_density', + 'participle_phrase_density', + 'appositive_phrase_density', + // Cohesion & Transitions + 'avg_transitions_per_sentence', + 'percent_sophisticated_transitions', + // Conceptual & Other + 'percent_sentences_w_one_concept', + 'percent_sentences_w_multi_concept', + 'percent_cleft_sentences', + 'max_clauses_in_any_sentence', + // Grades 5-12 + 'num_sentences', + 'num_simple_sentences', + 'num_compound', + 'num_basic_complex', + 'num_advanced_complex', + 'percentage_simple', + 'percentage_compound', + 'percentage_basic_complex', + 'percentage_advanced_complex', +] as const; + +/** + * Convert sentence features to JSON string for LLM prompt + * Ported from Python row_to_features_json + */ +export function featuresToJSON( + features: SentenceFeatures, + decimals = 1, + castToInt = true +): string { + const payload: Record = {}; + + for (const col of FEATURE_COLS) { + const value = features[col as keyof SentenceFeatures]; + if (typeof value === 'number') { + const rounded = Math.round(value * Math.pow(10, decimals)) / Math.pow(10, decimals); + payload[col] = castToInt ? Math.round(rounded) : rounded; + } else { + payload[col] = null; + } + } + + return JSON.stringify(payload, null, 2); +} diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index 16ac0e8..d8d0a08 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -33,6 +33,18 @@ export type { ProviderConfig, } from './providers/index.js'; +// Sentence structure exports +export type { + SentenceAnalysis, + ComplexityClassification, + SentenceFeatures, +} from './schemas/sentence-structure.js'; + +export { + SentenceAnalysisSchema, + ComplexityClassificationSchema, +} from './schemas/sentence-structure.js'; + // Vocabulary exports export type { VocabularyComplexity, @@ -43,6 +55,9 @@ export { VocabularyEvaluator, evaluateVocabulary, type VocabularyEvaluatorConfig, + SentenceStructureEvaluator, + evaluateSentenceStructure, + type SentenceStructureEvaluatorConfig, type BaseEvaluatorConfig, type TelemetryOptions, } from './evaluators/index.js'; @@ -51,5 +66,7 @@ export { export { calculateFleschKincaidGrade, calculateReadabilityMetrics, + addEngineeredFeatures, + featuresToJSON, type ReadabilityMetrics, } from './features/index.js'; diff --git a/sdks/typescript/src/prompts/sentence-structure/analysis.ts b/sdks/typescript/src/prompts/sentence-structure/analysis.ts new file mode 100644 index 0000000..c1fb7b7 --- /dev/null +++ b/sdks/typescript/src/prompts/sentence-structure/analysis.ts @@ -0,0 +1,22 @@ +import SYSTEM_PROMPT_ANALYSIS_TEMPLATE from '../../../../../evals/prompts/sentence-structure/analysis-system.txt'; +import USER_PROMPT_ANALYSIS_TEMPLATE from '../../../../../evals/prompts/sentence-structure/analysis-user.txt'; + +/** + * Get the system prompt for sentence grammatical analysis + * @returns The system prompt + */ +export function getSystemPromptAnalysis(): string { + return SYSTEM_PROMPT_ANALYSIS_TEMPLATE; +} + +/** + * Generate the user prompt for sentence analysis + * @param text - The text to analyze + * @param groundTruthCounts - Ground truth counts from readability metrics + * @returns The formatted user prompt + */ +export function getUserPromptAnalysis(text: string, groundTruthCounts: string): string { + return USER_PROMPT_ANALYSIS_TEMPLATE + .replace('{text}', text) + .replace('{ground_truth_counts}', groundTruthCounts); +} diff --git a/sdks/typescript/src/prompts/sentence-structure/complexity.ts b/sdks/typescript/src/prompts/sentence-structure/complexity.ts new file mode 100644 index 0000000..361a69a --- /dev/null +++ b/sdks/typescript/src/prompts/sentence-structure/complexity.ts @@ -0,0 +1,52 @@ +import SYSTEM_PROMPT_COMPLEXITY_TEMPLATE from '../../../../../evals/prompts/sentence-structure/complexity-system.txt'; +import USER_PROMPT_COMPLEXITY_TEMPLATE from '../../../../../evals/prompts/sentence-structure/complexity-user.txt'; +import RUBRIC_GRADE_3 from '../../../../../evals/prompts/sentence-structure/rubric-grade-3.txt'; +import RUBRIC_GRADE_4 from '../../../../../evals/prompts/sentence-structure/rubric-grade-4.txt'; +import RUBRIC_GRADES_5_12 from '../../../../../evals/prompts/sentence-structure/rubric-grades-5-12.txt'; + +/** + * Get the system prompt for sentence structure complexity evaluation + * @returns The system prompt + */ +export function getSystemPromptComplexity(): string { + return SYSTEM_PROMPT_COMPLEXITY_TEMPLATE; +} + +/** + * Get the appropriate rubric based on grade level + * @param grade - The target grade level (K-12) + * @returns The rubric text for the grade level + */ +export function getRubricForGrade(grade: string): string { + if (grade === '3') { + return RUBRIC_GRADE_3; + } else if (grade === '4') { + return RUBRIC_GRADE_4; + } else if (['5', '6', '7', '8', '9', '10', '11', '12'].includes(grade)) { + return RUBRIC_GRADES_5_12; + } else { + // K, 1, 2 - no specific rubric, use general principles + return 'No specific rubric available for this grade. Use general linguistic principles.'; + } +} + +/** + * Generate the user prompt for complexity evaluation + * @param sentenceFeatures - JSON string of sentence features + * @param grade - The target grade level + * @param excerpt - The original text excerpt + * @returns The formatted user prompt + */ +export function getUserPromptComplexity( + sentenceFeatures: string, + grade: string, + excerpt: string +): string { + const rubric = getRubricForGrade(grade); + + return USER_PROMPT_COMPLEXITY_TEMPLATE + .replace('{sentence_features}', sentenceFeatures) + .replace('{grade}', grade) + .replace('{rubric}', rubric) + .replace('{excerpt}', excerpt); +} diff --git a/sdks/typescript/src/prompts/sentence-structure/index.ts b/sdks/typescript/src/prompts/sentence-structure/index.ts new file mode 100644 index 0000000..af677ae --- /dev/null +++ b/sdks/typescript/src/prompts/sentence-structure/index.ts @@ -0,0 +1,6 @@ +export { getSystemPromptAnalysis, getUserPromptAnalysis } from './analysis.js'; +export { + getSystemPromptComplexity, + getUserPromptComplexity, + getRubricForGrade, +} from './complexity.js'; diff --git a/sdks/typescript/src/schemas/sentence-structure.ts b/sdks/typescript/src/schemas/sentence-structure.ts new file mode 100644 index 0000000..4f9522f --- /dev/null +++ b/sdks/typescript/src/schemas/sentence-structure.ts @@ -0,0 +1,124 @@ +import { z } from 'zod'; +import { ComplexityLevel } from './outputs.js'; + +/** + * Stage 1: Detailed sentence analysis output (40+ metrics) + * Ported from Python SentenceAnalysesEvaluatorOutput + */ +export const SentenceAnalysisSchema = z.object({ + reasoning: z.string().describe('Step-by-step reasoning for the analysis'), + + // Foundational + num_sentences: z.number().int().describe('Total number of sentences in the text'), + num_words: z.number().int().describe('Total number of words in the text'), + flesch_kincaid_grade: z.number().describe('Flesch-Kincaid Grade Level number'), + + // Sentence Type + num_simple_sentences: z.number().int().describe('Number of simple sentences'), + num_compound_sentences: z.number().int().describe('Number of compound sentences'), + num_complex_sentences: z.number().int().describe('Number of complex sentences'), + num_compound_complex_sentences: z.number().int().describe('Number of compound-complex sentences'), + num_other_sentences: z.number().int().describe('Number of other sentence types'), + + // Subordination + num_independent_clauses: z.number().int(), + num_subordinate_clauses: z.number().int(), + num_total_clauses: z.number().int(), + num_sentences_with_subordinate: z.number().int(), + num_sentences_with_multiple_subordinates: z.number().int(), + num_sentences_with_embedded_clauses: z.number().int(), + + // Informational Phrases + num_prepositional_phrases: z.number().int(), + num_participle_phrases: z.number().int(), + num_appositive_phrases: z.number().int(), + + // Cohesion + num_simple_transitions: z.number().int(), + num_sophisticated_transitions: z.number().int(), + + // Sentence Type Density + words_in_simple_sentences: z.number().int(), + words_in_compound_sentences: z.number().int(), + words_in_complex_sentences: z.number().int(), + words_in_compound_complex_sentences: z.number().int(), + words_in_other_sentences: z.number().int(), + + // Additional Features + sentence_word_counts: z.array(z.number().int()), + num_one_concept_sentences: z.number().int(), + num_multi_concept_sentences: z.number().int(), + num_cleft_sentences: z.number().int(), + max_clauses_in_any_sentence: z.number().int(), + + // Grades 5-12 specific + num_compound: z.number().int().describe('Number of compound sentences'), + num_basic_complex: z.number().int().describe('Number of basic complex sentences'), + num_advanced_complex: z.number().int().describe('Number of advanced complex sentences'), + percentage_simple: z.number().describe('Percentage of simple sentences'), + percentage_compound: z.number().describe('Percentage of compound sentences'), + percentage_basic_complex: z.number().describe('Percentage of basic complex sentences'), + percentage_advanced_complex: z.number().describe('Percentage of advanced complex sentences'), +}); + +export type SentenceAnalysis = z.infer; + +/** + * Stage 2: Final complexity classification + * Ported from Python ComplexityClassificationOutput + */ +export const ComplexityClassificationSchema = z.object({ + reasoning: z.string().describe('Detailed pedagogically appropriate reasoning'), + answer: ComplexityLevel, +}); + +export type ComplexityClassification = z.infer; + +/** + * Engineered features computed from sentence analysis + * These are calculated in TypeScript, not requested from LLM + */ +export interface SentenceFeatures extends SentenceAnalysis { + // Foundational & Distributional + avg_words_per_sentence: number; + sentence_length_variation: number; + percent_short_sentences: number; + percent_medium_sentences: number; + percent_long_sentences: number; + percent_very_long_sentences: number; + + // Sentence Structure (Grammatical Type) + percent_simple_sentences: number; + percent_compound_sentences: number; + percent_complex_sentences: number; + percent_compound_complex_sentences: number; + percent_other_sentences: number; + + // Word Distribution + percent_words_in_simple_sentences: number; + percent_words_in_complex_sentences: number; + percent_words_in_compound_sentences: number; + percent_words_in_compound_complex_sentences: number; + percent_words_in_other_sentences: number; + + // Clausal & Subordination + avg_subordinates_per_sentence: number; + avg_clauses_per_sentence: number; + percent_sentences_with_subordinate: number; + percent_sentences_with_multiple_subordinates: number; + percent_sentences_with_embedded_clauses: number; + + // Phrase Density + prep_phrase_density: number; + participle_phrase_density: number; + appositive_phrase_density: number; + + // Cohesion & Transitions + avg_transitions_per_sentence: number; + percent_sophisticated_transitions: number; + + // Conceptual & Other + percent_sentences_w_one_concept: number; + percent_sentences_w_multi_concept: number; + percent_cleft_sentences: number; +} diff --git a/sdks/typescript/tests/integration/sentence-structure.integration.test.ts b/sdks/typescript/tests/integration/sentence-structure.integration.test.ts new file mode 100644 index 0000000..dc802d4 --- /dev/null +++ b/sdks/typescript/tests/integration/sentence-structure.integration.test.ts @@ -0,0 +1,128 @@ +import { describe, it, expect, beforeAll } from 'vitest'; +import { SentenceStructureEvaluator } from '../../src/evaluators/sentence-structure.js'; +import { + runEvaluatorTest, + type BaseTestCase, +} from '../utils/index.js'; + +/** + * Sentence Structure Evaluator Integration Tests + * + * Test cases cover grades 2-6 with varying complexity levels. + * + * Each test uses a retry mechanism (up to 3 attempts) to account for LLM non-determinism, + * with short-circuiting on first expected match. If no expected match is found after all + * attempts, the test checks if any result falls within the acceptable value range. + * + * To run these tests: + * ```bash + * RUN_INTEGRATION_TESTS=true npm run test:integration + * ``` + */ + +const SKIP_INTEGRATION = !process.env.RUN_INTEGRATION_TESTS && + !process.env.OPENAI_API_KEY; + +const describeIntegration = SKIP_INTEGRATION ? describe.skip : describe; + +// Test timeout: 2 minutes per test case (allows for 3 attempts with API latency) +const TEST_TIMEOUT_MS = 2 * 60 * 1000; + +const TEST_CASES: BaseTestCase[] = [ + { + id: 'SS2', + grade: '2', + text: "The Roman Empire was a powerful empire that lasted for hundreds of years. It started as a small village in Italy and grew into a huge empire that controlled much of Europe, Asia, and Africa. The Roman Empire had many strong leaders like Julius Caesar and Augustus. These leaders helped the empire grow and become very powerful.\n \n\n The Roman Empire had a period of peace and prosperity called the Pax Romana. This time was good for the empire, but it didn't last forever. The empire started to have problems. The army became weaker, and the economy had problems. The empire was also attacked by groups of people called barbarians.\n \n\n The Roman Empire was divided into two parts: the Western Roman Empire and the Eastern Roman Empire. The Western Roman Empire eventually fell apart in 476 AD. The Eastern Roman Empire, also known as the Byzantine Empire, lasted for many more years. The Roman Empire left behind many things that we still use today, like the Roman alphabet and the calendar.", + expected: 'moderately complex', + acceptable: ['slightly complex', 'very complex'], + }, + { + id: 'SS3', + grade: '3', + text: "The hoisting gear consists of a double system of chains 13/16 in. in diameter placed side by side; each chain is anchored by an adjustable screw to the end of the jib, and, passing round the traveling carriage and down to the falling block, is taken along the jib over a sliding pulley which leads it on to the grooved barrel, 3 ft. 9 in. in diameter. In front of the barrel is placed an automatic winder which insures a proper coiling of the chain in the grooves. The motive power is derived from two cylinders 10 in. in diameter and 16 in. stroke, one being bolted to each side frame; these cylinders, which are provided with link motion and reversing gear, drive a steel crank shaft 2¾ in. in diameter; on this shaft is a steel sliding pinion which drives the barrel by a double purchase.", + expected: 'exceedingly complex', + acceptable: ['very complex'], + }, + { + id: 'SS4', + grade: '4', + text: "Before corals bleach, they do not show many other signs of feeling stressed. So, if we want to understand a coral's health, we have to study its cells. Inside cells we have a lot of information, including DNA, RNA, and proteins. These molecules can help us find clues about the communication between the coral and the algae. But also, these molecules can teach us how to know when corals are stressed.\nWhen an organism is stressed, every cell in its body will react. Everything will do its best to survive! In response to stress, the cell will use its DNA to make RNA, so that it can then make proteins that will fight off the stress. If an organism has been stressed before, it can respond to the stress faster and better. Think of it like visiting a city: the first time you visit, you will need a map to find your hotel. The more often you visit the city, the less you will need the map because you will remember, and you will get back to the hotel faster.", + expected: 'very complex', + acceptable: ['moderately complex', 'exceedingly complex'], + }, + { + id: 'SS5', + grade: '5', + text: "Mesopotamia, located in present-day Iraq, is known as the 'Cradle of Civilization' because it was home to some of the earliest civilizations in the world. The region got its name from the ancient Greek words for 'land between the rivers,' referring to the Tigris and Euphrates rivers. These rivers provided water for the fertile land, making it perfect for farming. The regular flooding of the rivers made the land around them ideal for growing crops, which helped people settle down and form permanent villages. These villages eventually grew into cities, where people developed many of the characteristics of civilization, like organized government, complex buildings, and different social classes.\n \n\n The first civilizations in Mesopotamia were the Sumerians, who lived around 5,000 years ago. They invented the world's first written language, called cuneiform, which they used to keep track of things like food supplies and trade. They also developed a system of numbers, which helped them with math and measurement. The Sumerians built impressive cities like Ur, Eridu, and Uruk, which had populations of over 50,000 people. These cities were centers of learning and culture, and they helped spread knowledge and ideas throughout the region.\n \n\n Over time, other civilizations rose and fell in Mesopotamia, including the Akkadians, Babylonians, and Assyrians. Each civilization made its own contributions to the development of human society. The Babylonians are famous for their code of laws, which was one of the first written legal systems in the world. The Assyrians were known for their powerful military and their impressive palaces. Mesopotamia's history is full of amazing inventions and innovations that shaped the world we live in today.\n \n\n The development of civilization in Mesopotamia was not just about the fertile land and the rivers. Changes in climate and the environment also played a role. People had to become more organized and work together to survive. This led to the development of complex societies and governments. Mesopotamia's story is a reminder of how human ingenuity and adaptability can lead to amazing achievements.\n \n\n The 'Cradle of Civilization' is a term that refers to the regions where the earliest known human civilizations emerged. Mesopotamia is a prime example of this, as it was a place where people learned to live together, build cities, and develop new technologies that changed the course of human history. The innovations that came from Mesopotamia, like writing, mathematics, and agriculture, continue to influence our lives today. By studying ancient Mesopotamia, we can learn about the origins of our own civilization and the challenges and triumphs of early humans.", + expected: 'slightly complex', + acceptable: ['moderately complex'], + // TODO: Valiadate the test-case with additional data from Grade 5 + // expected: 'exceedingly complex', + // acceptable: ['very complex'], + }, + { + id: 'SS6', + grade: '6', + text: "Benjamin Franklin was a very important person in American history. He was born in Boston, Massachusetts in 1706. He was one of 17 children. Franklin did not go to school for very long. He learned to be a printer from his brother. Franklin was a very smart man. He invented many things, like bifocals, the Franklin stove, and the lightning rod. He also started the first public library in Philadelphia. Franklin was a writer, too. He wrote a book called *Poor Richard's Almanack*. It had many famous sayings, like \"Lost Time is never found again.\"\n\nFranklin was also a politician. He helped write the Declaration of Independence. He was a diplomat, too. He helped the United States get help from France during the Revolutionary War. He was a very busy man! Franklin was a scientist, a writer, a politician, and an inventor. He was a very important person in American history.\n\nFranklin was a very interesting person. He was a scientist who did experiments with electricity. He was a writer who wrote a book of sayings. He was a politician who helped the United States become independent. He was a diplomat who helped the United States get help from other countries. He was a very busy man!\n\nFranklin was a very smart man. He was a self-taught man who learned a lot on his own. He was a very creative man who invented many things. He was a very kind man who helped others. He was a very important man who helped shape the United States.\n\nFranklin was a very influential person. He was a leader who helped people. He was a thinker who came up with new ideas. He was a writer who shared his thoughts with others. He was a scientist who helped people understand the world. He was a very important person who helped make the United States what it is today.", + expected: 'slightly complex', + acceptable: ['moderately complex'], + }, +]; + +describeIntegration.concurrent('Sentence Structure Evaluator - Comprehensive Test Suite', () => { + let evaluator: SentenceStructureEvaluator; + + beforeAll(() => { + if (SKIP_INTEGRATION) { + console.log('⏭️ Skipping integration tests (no API keys or RUN_INTEGRATION_TESTS not set)'); + return; + } + + evaluator = new SentenceStructureEvaluator({ + openaiApiKey: process.env.OPENAI_API_KEY!, + }); + + console.log('\n' + '='.repeat(80)); + console.log('SENTENCE STRUCTURE EVALUATOR - TEST SUITE (PARALLEL)'); + console.log('='.repeat(80)); + console.log(`Running ${TEST_CASES.length} test cases with up to 3 attempts each`); + console.log('Short-circuiting on first expected match'); + console.log('Checking acceptable values if no expected match'); + console.log('='.repeat(80)); + }); + + // Generate individual test for each case + TEST_CASES.forEach((testCase) => { + it.concurrent(`${testCase.id}: Grade ${testCase.grade} - ${testCase.expected}`, async () => { + // Buffer all logs to print atomically at the end (prevents interleaving in parallel tests) + const logBuffer: string[] = []; + + // Test header + logBuffer.push('\n' + '='.repeat(80)); + logBuffer.push(`Test Case ${testCase.id} | Grade: ${testCase.grade}`); + logBuffer.push('='.repeat(80)); + logBuffer.push(`Expected Complexity: ${testCase.expected}`); + logBuffer.push(`Text Preview: ${testCase.text.substring(0, 100)}...`); + logBuffer.push(''); + + // Run the evaluation (returns logs instead of printing) + const maxAttempts = 3; + const result = await runEvaluatorTest(testCase, { + evaluator, + extractResult: (r) => r.score, + maxAttempts, + }); + + // Add evaluation logs to buffer (includes detailed summary) + logBuffer.push(...result.logs); + + // Print all logs atomically at the end - single console.log to prevent interleaving + console.log(logBuffer.join('\n')); + + // Assert that we got a match within maxAttempts (expected or acceptable) + expect(result.matched).toBe(true); + expect(result.matchedOnAttempt).toBeDefined(); + expect(result.matchedOnAttempt).toBeLessThanOrEqual(maxAttempts); + }, TEST_TIMEOUT_MS); + }); +}); diff --git a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts new file mode 100644 index 0000000..96afdc5 --- /dev/null +++ b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts @@ -0,0 +1,249 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { SentenceStructureEvaluator } from '../../../src/evaluators/sentence-structure.js'; +import { ConfigurationError } from '../../../src/errors.js'; +import type { LLMProvider } from '../../../src/providers/base.js'; + +/** + * Comprehensive unit tests for SentenceStructureEvaluator + * + * These tests verify: + * - Constructor validation + * - Successful evaluation flow (both stages) + * - Error handling (LLM failures) + * - Telemetry behavior + * - Response structure + */ + +// Helper to create minimal valid sentence analysis mock +const createMockSentenceAnalysis = () => ({ + reasoning: 'Mock analysis', + num_sentences: 2, + num_words: 10, + flesch_kincaid_grade: 3.5, + num_simple_sentences: 2, + num_compound_sentences: 0, + num_complex_sentences: 0, + num_compound_complex_sentences: 0, + num_other_sentences: 0, + num_independent_clauses: 2, + num_subordinate_clauses: 0, + num_total_clauses: 2, + num_sentences_with_subordinate: 0, + num_sentences_with_multiple_subordinates: 0, + num_sentences_with_embedded_clauses: 0, + num_prepositional_phrases: 1, + num_participle_phrases: 0, + num_appositive_phrases: 0, + num_simple_transitions: 0, + num_sophisticated_transitions: 0, + words_in_simple_sentences: 10, + words_in_compound_sentences: 0, + words_in_complex_sentences: 0, + words_in_compound_complex_sentences: 0, + words_in_other_sentences: 0, + sentence_word_counts: [5, 5], + num_one_concept_sentences: 2, + num_multi_concept_sentences: 0, + num_cleft_sentences: 0, + max_clauses_in_any_sentence: 1, + num_compound: 0, + num_basic_complex: 0, + num_advanced_complex: 0, + percentage_simple: 100, + percentage_compound: 0, + percentage_basic_complex: 0, + percentage_advanced_complex: 0, +}); + +// Mock providers +const createMockProvider = (): LLMProvider => ({ + generateStructured: vi.fn(), + generateText: vi.fn(), +}); + +// Mock the createProvider factory +vi.mock('../../../src/providers/index.js', () => ({ + createProvider: vi.fn(() => createMockProvider()), +})); + +// Mock telemetry to avoid real HTTP calls +vi.mock('../../../src/telemetry/client.js', () => { + return { + TelemetryClient: class MockTelemetryClient { + send = vi.fn().mockResolvedValue(undefined); + }, + }; +}); + +describe('SentenceStructureEvaluator - Constructor Validation', () => { + it('should throw ConfigurationError when OpenAI API key is missing', () => { + expect(() => new SentenceStructureEvaluator({ + openaiApiKey: '', + })).toThrow(ConfigurationError); + }); + +}); + +describe('SentenceStructureEvaluator - Evaluation Flow', () => { + let evaluator: SentenceStructureEvaluator; + let mockAnalysisProvider: LLMProvider; + let mockComplexityProvider: LLMProvider; + + beforeEach(() => { + vi.clearAllMocks(); + + // Create evaluator (providers will be mocked) + evaluator = new SentenceStructureEvaluator({ + openaiApiKey: 'test-openai-key', + telemetry: false, + }); + + // Get references to the mocked providers + // @ts-expect-error Accessing private property for testing + mockAnalysisProvider = evaluator.analysisProvider; + // @ts-expect-error Accessing private property for testing + mockComplexityProvider = evaluator.complexityProvider; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('Successful Evaluation Flow', () => { + it('should successfully evaluate text through both stages', async () => { + const testText = 'The cat sat on the mat. It was sleeping peacefully.'; + const testGrade = 'K'; + + // Mock sentence analysis response + vi.mocked(mockAnalysisProvider.generateStructured).mockResolvedValue({ + data: createMockSentenceAnalysis(), + model: 'gpt-4o', + usage: { + inputTokens: 150, + outputTokens: 100, + }, + latencyMs: 600, + }); + + // Mock complexity classification response + vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ + data: { + answer: 'Slightly Complex', + reasoning: 'The text uses simple sentence structures appropriate for kindergarten.', + }, + model: 'gpt-4o', + usage: { + inputTokens: 250, + outputTokens: 80, + }, + latencyMs: 500, + }); + + // Execute evaluation + const result = await evaluator.evaluate(testText, testGrade); + + // Verify result structure + expect(result.score).toBe('Slightly Complex'); + expect(result.reasoning).toContain('simple sentence structures'); + expect(result.metadata).toBeDefined(); + expect(result.metadata.model).toBe('openai:gpt-4o'); + expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); + + // Verify both providers were called + expect(mockAnalysisProvider.generateStructured).toHaveBeenCalledTimes(1); + expect(mockComplexityProvider.generateStructured).toHaveBeenCalledTimes(1); + + // Verify internal data structure + expect(result._internal).toBeDefined(); + expect(result._internal?.sentenceAnalysis).toBeDefined(); + expect(result._internal?.features).toBeDefined(); + expect(result._internal?.complexity).toBeDefined(); + }); + }); + + describe('Error Handling', () => { + it('should handle sentence analysis API failure', async () => { + const testText = 'Test text here for API failure'; + const testGrade = '3'; + + // Mock analysis failure + vi.mocked(mockAnalysisProvider.generateStructured).mockRejectedValue( + new Error('API timeout') + ); + + // Should propagate the error + await expect(evaluator.evaluate(testText, testGrade)) + .rejects.toThrow('API timeout'); + + // Verify complexity provider was never called + expect(mockComplexityProvider.generateStructured).not.toHaveBeenCalled(); + }); + + it('should handle complexity classification API failure', async () => { + const testText = 'Test text here for complexity failure'; + const testGrade = '5'; + + // Mock successful analysis + vi.mocked(mockAnalysisProvider.generateStructured).mockResolvedValue({ + data: createMockSentenceAnalysis(), + model: 'gpt-4o', + usage: { inputTokens: 150, outputTokens: 100 }, + latencyMs: 600, + }); + + // Mock complexity failure + vi.mocked(mockComplexityProvider.generateStructured).mockRejectedValue( + new Error('Schema validation failed') + ); + + // Should propagate the error + await expect(evaluator.evaluate(testText, testGrade)) + .rejects.toThrow('Schema validation failed'); + + // Verify analysis provider was called (stage 1 completed) + expect(mockAnalysisProvider.generateStructured).toHaveBeenCalledTimes(1); + }); + + }); + + describe('Response Structure', () => { + it('should return correct result structure', async () => { + vi.mocked(mockAnalysisProvider.generateStructured).mockResolvedValue({ + data: createMockSentenceAnalysis(), + model: 'gpt-4o', + usage: { inputTokens: 150, outputTokens: 100 }, + latencyMs: 600, + }); + + vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ + data: { + answer: 'Moderately Complex', + reasoning: 'Detailed reasoning here', + }, + model: 'gpt-4o', + usage: { inputTokens: 250, outputTokens: 80 }, + latencyMs: 500, + }); + + const result = await evaluator.evaluate('Test text here', '5'); + + // Verify result structure + expect(result).toHaveProperty('score'); + expect(result).toHaveProperty('reasoning'); + expect(result).toHaveProperty('metadata'); + expect(result).toHaveProperty('_internal'); + + // Verify metadata structure + expect(result.metadata).toHaveProperty('promptVersion'); + expect(result.metadata).toHaveProperty('model'); + expect(result.metadata).toHaveProperty('timestamp'); + expect(result.metadata).toHaveProperty('processingTimeMs'); + + // Verify metadata values + expect(result.metadata.promptVersion).toBe('1.2.0'); + expect(result.metadata.model).toBe('openai:gpt-4o'); + expect(result.metadata.timestamp).toBeInstanceOf(Date); + expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); + }); + }); +}); From 2da9103ca6fa591b251e9bf689f178fa700b903d Mon Sep 17 00:00:00 2001 From: Adnan Hussain <61515575+adnanrhussain@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:20:32 -0800 Subject: [PATCH 5/9] feat: Implement GLA evaluator (#14) * feat: Implement GLA evaluator --- sdks/typescript/README.md | 46 ++++ .../evaluators/grade-level-appropriateness.ts | 201 ++++++++++++++++++ sdks/typescript/src/evaluators/index.ts | 6 + sdks/typescript/src/index.ts | 11 + .../grade-level-appropriateness/index.ts | 21 ++ .../prompts/sentence-structure/analysis.ts | 3 +- .../prompts/sentence-structure/complexity.ts | 3 +- .../schemas/grade-level-appropriateness.ts | 27 +++ sdks/typescript/src/schemas/index.ts | 5 + .../grade-level-appropriateness.test.ts | 185 ++++++++++++++++ 10 files changed, 506 insertions(+), 2 deletions(-) create mode 100644 sdks/typescript/src/evaluators/grade-level-appropriateness.ts create mode 100644 sdks/typescript/src/prompts/grade-level-appropriateness/index.ts create mode 100644 sdks/typescript/src/schemas/grade-level-appropriateness.ts create mode 100644 sdks/typescript/tests/unit/evaluators/grade-level-appropriateness.test.ts diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md index 1fc51cb..3965e82 100644 --- a/sdks/typescript/README.md +++ b/sdks/typescript/README.md @@ -121,6 +121,52 @@ await evaluator.evaluate(text: string, grade: string) --- +### 3. Grade Level Appropriateness Evaluator + +Determines appropriate grade level for text. + +**No grade parameter required** - evaluates what grade the text is appropriate for. + +**Uses:** Google Gemini 2.5 Pro + +**Constructor:** +```typescript +const evaluator = new GradeLevelAppropriatenessEvaluator({ + googleApiKey: string; // Required - Google API key + maxRetries?: number; // Optional - Max retry attempts (default: 2) + telemetry?: boolean | TelemetryOptions; // Optional (default: true) + logger?: Logger; // Optional - Custom logger + logLevel?: LogLevel; // Optional - Logging verbosity (default: WARN) +}); +``` + +**API:** +```typescript +await evaluator.evaluate(text: string) +``` + +**Returns:** +```typescript +{ + score: string; // e.g., 'K-1', '2-3', '4-5', '6-8', '9-10', '11-CCR' + reasoning: string; + metadata: { + promptVersion: string; + model: string; + timestamp: Date; + processingTimeMs: number; + }; + _internal: { + grade: string; + alternative_grade: string; + scaffolding_needed: string; + reasoning: string; + }; +} +``` + +--- + ## Error Handling The SDK provides specific error types to help you handle different scenarios: diff --git a/sdks/typescript/src/evaluators/grade-level-appropriateness.ts b/sdks/typescript/src/evaluators/grade-level-appropriateness.ts new file mode 100644 index 0000000..525e162 --- /dev/null +++ b/sdks/typescript/src/evaluators/grade-level-appropriateness.ts @@ -0,0 +1,201 @@ +import type { LLMProvider } from '../providers/index.js'; +import { createProvider } from '../providers/index.js'; +import { + GradeLevelAppropriatenessSchema, + type GradeLevelAppropriateness, +} from '../schemas/grade-level-appropriateness.js'; +import { getSystemPrompt, getUserPrompt } from '../prompts/grade-level-appropriateness/index.js'; +import type { EvaluationResult } from '../schemas/index.js'; +import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; +import { ConfigurationError, ValidationError, wrapProviderError } from '../errors.js'; + +/** + * Configuration for GradeLevelAppropriatenessEvaluator + */ +export interface GradeLevelAppropriatenessEvaluatorConfig extends BaseEvaluatorConfig { + /** Google API key for grade level evaluation (uses Gemini 2.5 Pro) */ + googleApiKey: string; +} + +/** + * Grade Level Appropriateness Evaluator + * + * Evaluates whether AI-generated text is suitable for a given grade band. + * Uses a structured 4-step analysis process: + * 1. Quantitative analysis (word count, Flesch-Kincaid) + * 2. Qualitative complexity (text structure, language, purpose, knowledge demands) + * 3. Background knowledge assessment + * 4. Synthesis and final recommendation + * + * Returns: + * - Target grade band (K-1, 2-3, 4-5, 6-8, 9-10, 11-CCR) + * - Alternative grade band (with scaffolding) + * - Specific scaffolding recommendations + * + * @example + * ```typescript + * const evaluator = new GradeLevelAppropriatenessEvaluator({ + * googleApiKey: process.env.GOOGLE_API_KEY + * }); + * + * const result = await evaluator.evaluate(text); + * console.log(result.score); // "9-10" + * console.log(result._internal.alternative_grade); // "6-8" + * console.log(result._internal.scaffolding_needed); + * ``` + */ +export class GradeLevelAppropriatenessEvaluator extends BaseEvaluator { + private provider: LLMProvider; + + constructor(config: GradeLevelAppropriatenessEvaluatorConfig) { + // Call base constructor for common setup (telemetry, etc.) + super(config); + + // Validate required API keys + if (!config.googleApiKey) { + throw new ConfigurationError('Google API key is required. Pass googleApiKey in config.'); + } + + // Create Google Gemini provider + this.provider = createProvider({ + type: 'google', + model: 'gemini-2.5-pro', + apiKey: config.googleApiKey, + maxRetries: this.config.maxRetries, + }); + } + + // Implement abstract methods from BaseEvaluator + protected getEvaluatorType(): string { + return 'grade-level-appropriateness'; + } + + /** + * Evaluate grade level appropriateness for a given text + * + * @param text - The text to evaluate + * @returns Evaluation result with grade recommendations and scaffolding suggestions + * @throws {ValidationError} If text is empty or too short/long + * @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError) + */ + async evaluate(text: string): Promise> { + this.logger.info('Starting grade level appropriateness evaluation', { + evaluator: 'grade-level-appropriateness', + operation: 'evaluate', + textLength: text.length, + }); + + const startTime = Date.now(); + + try { + // Validate inputs — inside try so validation errors are telemetered. + this.validateText(text); + this.logger.debug('Evaluating grade level appropriateness', { + evaluator: 'grade-level-appropriateness', + operation: 'grade_evaluation', + }); + const userPrompt = getUserPrompt(text); + + const response = await this.provider.generateStructured({ + messages: [ + { role: 'system', content: getSystemPrompt() }, + { role: 'user', content: userPrompt }, + ], + schema: GradeLevelAppropriatenessSchema, + temperature: 0.25, + }); + + const latencyMs = Date.now() - startTime; + + const tokenUsage = { + input_tokens: response.usage.inputTokens, + output_tokens: response.usage.outputTokens, + }; + + const result = { + score: response.data.grade, + reasoning: response.data.reasoning, + metadata: { + promptVersion: '1.2.0', + model: 'google:gemini-2.5-pro', + timestamp: new Date(), + processingTimeMs: latencyMs, + }, + _internal: response.data, + }; + + // Send success telemetry (fire-and-forget) + this.sendTelemetry({ + status: 'success', + latencyMs, + textLength: text.length, + provider: 'google:gemini-2.5-pro', + tokenUsage, + // No metadata.stage_details for single-stage evaluator + inputText: text, + }).catch(() => { + // Ignore telemetry errors + }); + + this.logger.info('Grade level appropriateness evaluation completed successfully', { + evaluator: 'grade-level-appropriateness', + operation: 'evaluate', + grade: result.score, + processingTimeMs: latencyMs, + }); + + return result; + } catch (error) { + const latencyMs = Date.now() - startTime; + + // Log the error + this.logger.error('Grade level appropriateness evaluation failed', { + evaluator: 'grade-level-appropriateness', + operation: 'evaluate', + error: error instanceof Error ? error : undefined, + processingTimeMs: latencyMs, + }); + + // Send failure telemetry (fire-and-forget) + this.sendTelemetry({ + status: 'error', + latencyMs, + textLength: text.length, + provider: 'google:gemini-2.5-pro', + errorCode: error instanceof Error ? error.name : 'UnknownError', + inputText: text, + }).catch(() => { + // Ignore telemetry errors + }); + + // Re-throw validation errors as-is + if (error instanceof ValidationError) { + throw error; + } + + // Wrap provider errors into appropriate error types + throw wrapProviderError(error, 'Grade level appropriateness evaluation failed'); + } + } +} + +/** + * Functional API for grade level appropriateness evaluation + * + * @example + * ```typescript + * const result = await evaluateGradeLevelAppropriateness( + * "Tides are the rise and fall of sea levels...", + * { + * googleApiKey: process.env.GOOGLE_API_KEY + * } + * ); + * ``` + */ +export async function evaluateGradeLevelAppropriateness( + text: string, + config: GradeLevelAppropriatenessEvaluatorConfig +): Promise> { + const evaluator = new GradeLevelAppropriatenessEvaluator(config); + return evaluator.evaluate(text); +} diff --git a/sdks/typescript/src/evaluators/index.ts b/sdks/typescript/src/evaluators/index.ts index e96898e..12b55d6 100644 --- a/sdks/typescript/src/evaluators/index.ts +++ b/sdks/typescript/src/evaluators/index.ts @@ -11,3 +11,9 @@ export { evaluateSentenceStructure, type SentenceStructureEvaluatorConfig, } from './sentence-structure.js'; + +export { + GradeLevelAppropriatenessEvaluator, + evaluateGradeLevelAppropriateness, + type GradeLevelAppropriatenessEvaluatorConfig, +} from './grade-level-appropriateness.js'; diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index d8d0a08..a84fcc0 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -7,6 +7,8 @@ export type { EvaluationError, } from './schemas/index.js'; +export { ComplexityLevel, GradeLevel, GradeBand } from './schemas/index.js'; + // Error types export { EvaluatorError, @@ -51,6 +53,12 @@ export type { VocabularyComplexityLevel, } from './schemas/vocabulary.js'; +// Grade Level Appropriateness exports +export type { GradeLevelAppropriateness } from './schemas/grade-level-appropriateness.js'; + +export { GradeLevelAppropriatenessSchema } from './schemas/grade-level-appropriateness.js'; + + export { VocabularyEvaluator, evaluateVocabulary, @@ -58,6 +66,9 @@ export { SentenceStructureEvaluator, evaluateSentenceStructure, type SentenceStructureEvaluatorConfig, + GradeLevelAppropriatenessEvaluator, + evaluateGradeLevelAppropriateness, + type GradeLevelAppropriatenessEvaluatorConfig, type BaseEvaluatorConfig, type TelemetryOptions, } from './evaluators/index.js'; diff --git a/sdks/typescript/src/prompts/grade-level-appropriateness/index.ts b/sdks/typescript/src/prompts/grade-level-appropriateness/index.ts new file mode 100644 index 0000000..4192b62 --- /dev/null +++ b/sdks/typescript/src/prompts/grade-level-appropriateness/index.ts @@ -0,0 +1,21 @@ +import SYSTEM_PROMPT_TEMPLATE from '../../../../../evals/prompts/grade-level-appropriateness/system.txt'; +import USER_PROMPT_TEMPLATE from '../../../../../evals/prompts/grade-level-appropriateness/user.txt'; + +/** + * Get the system prompt for grade level appropriateness evaluation + * @returns The system prompt + */ +export function getSystemPrompt(): string { + return SYSTEM_PROMPT_TEMPLATE; +} + +/** + * Get the user prompt with the text to evaluate + * @param text - The text to evaluate for grade level appropriateness + * @returns The formatted user prompt + */ +export function getUserPrompt(text: string): string { + return USER_PROMPT_TEMPLATE + .replace('{text}', text) + .replace('{format_instructions}', ''); +} diff --git a/sdks/typescript/src/prompts/sentence-structure/analysis.ts b/sdks/typescript/src/prompts/sentence-structure/analysis.ts index c1fb7b7..f5e8c7f 100644 --- a/sdks/typescript/src/prompts/sentence-structure/analysis.ts +++ b/sdks/typescript/src/prompts/sentence-structure/analysis.ts @@ -18,5 +18,6 @@ export function getSystemPromptAnalysis(): string { export function getUserPromptAnalysis(text: string, groundTruthCounts: string): string { return USER_PROMPT_ANALYSIS_TEMPLATE .replace('{text}', text) - .replace('{ground_truth_counts}', groundTruthCounts); + .replace('{ground_truth_counts}', groundTruthCounts) + .replace('{format_instructions}', ''); } diff --git a/sdks/typescript/src/prompts/sentence-structure/complexity.ts b/sdks/typescript/src/prompts/sentence-structure/complexity.ts index 361a69a..32189ea 100644 --- a/sdks/typescript/src/prompts/sentence-structure/complexity.ts +++ b/sdks/typescript/src/prompts/sentence-structure/complexity.ts @@ -48,5 +48,6 @@ export function getUserPromptComplexity( .replace('{sentence_features}', sentenceFeatures) .replace('{grade}', grade) .replace('{rubric}', rubric) - .replace('{excerpt}', excerpt); + .replace('{excerpt}', excerpt) + .replace('{format_instructions}', ''); } diff --git a/sdks/typescript/src/schemas/grade-level-appropriateness.ts b/sdks/typescript/src/schemas/grade-level-appropriateness.ts new file mode 100644 index 0000000..e23e638 --- /dev/null +++ b/sdks/typescript/src/schemas/grade-level-appropriateness.ts @@ -0,0 +1,27 @@ +import { z } from 'zod'; + +/** + * Valid grade bands for grade level appropriateness evaluation + */ +export const GradeBand = z.enum(['K-1', '2-3', '4-5', '6-8', '9-10', '11-CCR']); + +export type GradeBand = z.infer; + +/** + * Output schema for Grade Level Appropriateness evaluation + * Matches Python OutputRanges model + */ +export const GradeLevelAppropriatenessSchema = z.object({ + reasoning: z + .string() + .describe( + 'Your reasoning for your answer in numbered bullet points for 4 steps with a 4th bullet point for synthesis.' + ), + grade: GradeBand.describe('The appropriate grade level for the text'), + alternative_grade: GradeBand.describe('An alternative grade level for the text'), + scaffolding_needed: z + .string() + .describe('Scaffolding needed for the text to be appropriate for the alternative grade'), +}); + +export type GradeLevelAppropriateness = z.infer; diff --git a/sdks/typescript/src/schemas/index.ts b/sdks/typescript/src/schemas/index.ts index ded6c72..f1b73d3 100644 --- a/sdks/typescript/src/schemas/index.ts +++ b/sdks/typescript/src/schemas/index.ts @@ -8,3 +8,8 @@ export { type EvaluationError, } from './outputs.js'; +export { + GradeBand, + GradeLevelAppropriatenessSchema, + type GradeLevelAppropriateness, +} from './grade-level-appropriateness.js'; diff --git a/sdks/typescript/tests/unit/evaluators/grade-level-appropriateness.test.ts b/sdks/typescript/tests/unit/evaluators/grade-level-appropriateness.test.ts new file mode 100644 index 0000000..7e04d9b --- /dev/null +++ b/sdks/typescript/tests/unit/evaluators/grade-level-appropriateness.test.ts @@ -0,0 +1,185 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { GradeLevelAppropriatenessEvaluator } from '../../../src/evaluators/grade-level-appropriateness.js'; +import { ConfigurationError, ValidationError } from '../../../src/errors.js'; +import type { LLMProvider } from '../../../src/providers/base.js'; + +/** + * Comprehensive unit tests for GradeLevelAppropriatenessEvaluator + * + * These tests verify: + * - Constructor validation + * - Successful evaluation flow (single stage) + * - Error handling (LLM failures) + * - Telemetry behavior + * - Response structure + */ + +// Mock providers +const createMockProvider = (): LLMProvider => ({ + generateStructured: vi.fn(), + generateText: vi.fn(), +}); + +// Mock the createProvider factory +vi.mock('../../../src/providers/index.js', () => ({ + createProvider: vi.fn(() => createMockProvider()), +})); + +// Mock telemetry to avoid real HTTP calls +vi.mock('../../../src/telemetry/client.js', () => { + return { + TelemetryClient: class MockTelemetryClient { + send = vi.fn().mockResolvedValue(undefined); + }, + }; +}); + +describe('GradeLevelAppropriatenessEvaluator - Constructor Validation', () => { + it('should throw ConfigurationError when Google API key is missing', () => { + expect(() => new GradeLevelAppropriatenessEvaluator({ + googleApiKey: '', + })).toThrow(ConfigurationError); + }); + +}); + +describe('GradeLevelAppropriatenessEvaluator - Evaluation Flow', () => { + let evaluator: GradeLevelAppropriatenessEvaluator; + let mockProvider: LLMProvider; + + beforeEach(() => { + vi.clearAllMocks(); + + // Create evaluator (provider will be mocked) + evaluator = new GradeLevelAppropriatenessEvaluator({ + googleApiKey: 'test-google-key', + telemetry: false, + }); + + // Get reference to the mocked provider + // @ts-expect-error Accessing private property for testing + mockProvider = evaluator.provider; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('Successful Evaluation Flow', () => { + it('should successfully evaluate text', async () => { + const testText = 'Tides are the rise and fall of sea levels caused by the combined effects of the gravitational forces exerted by the Moon and the Sun.'; + + // Mock grade level response + vi.mocked(mockProvider.generateStructured).mockResolvedValue({ + data: { + grade: '6-8', + alternative_grade: '4-5', + scaffolding_needed: 'Pre-teach gravitational forces; Use visual diagrams of moon-sun-earth system', + reasoning: 'The text discusses gravitational forces and celestial mechanics, which are appropriate for middle school science curriculum.', + }, + model: 'gemini-2.5-pro', + usage: { + inputTokens: 200, + outputTokens: 150, + }, + latencyMs: 800, + }); + + // Execute evaluation (no grade parameter needed) + const result = await evaluator.evaluate(testText); + + // Verify result structure + expect(result.score).toBe('6-8'); + expect(result._internal).toBeDefined(); + expect(result._internal!.grade).toBe('6-8'); + expect(result._internal!.alternative_grade).toBe('4-5'); + expect(result._internal!.scaffolding_needed).toContain('gravitational forces'); + expect(result.reasoning).toContain('gravitational forces'); + expect(result.metadata).toBeDefined(); + expect(result.metadata.model).toBe('google:gemini-2.5-pro'); + expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); + + // Verify provider was called + expect(mockProvider.generateStructured).toHaveBeenCalledTimes(1); + }); + + }); + + describe('Input Validation', () => { + it('should throw ValidationError for empty text', async () => { + await expect(evaluator.evaluate('')) + .rejects.toThrow(ValidationError); + }); + + it('should throw ValidationError for text that is too short', async () => { + await expect(evaluator.evaluate('Hi')) + .rejects.toThrow(ValidationError); + }); + }); + + describe('Error Handling', () => { + it('should handle API failure', async () => { + const testText = 'Test text here for API failure'; + + // Mock API failure + vi.mocked(mockProvider.generateStructured).mockRejectedValue( + new Error('API timeout') + ); + + // Should propagate the error + await expect(evaluator.evaluate(testText)) + .rejects.toThrow('API timeout'); + }); + + }); + + describe('Response Structure', () => { + it('should return correct result structure', async () => { + vi.mocked(mockProvider.generateStructured).mockResolvedValue({ + data: { + grade: '9-10', + alternative_grade: '6-8', + scaffolding_needed: 'Pre-teach advanced vocabulary; Provide background context', + reasoning: 'Detailed reasoning about grade appropriateness', + }, + model: 'gemini-2.5-pro', + usage: { inputTokens: 200, outputTokens: 150 }, + latencyMs: 800, + }); + + const result = await evaluator.evaluate('Test text here'); + + // Verify result structure + expect(result).toHaveProperty('score'); + expect(result).toHaveProperty('reasoning'); + expect(result).toHaveProperty('metadata'); + expect(result).toHaveProperty('_internal'); + + // Verify score is the grade string + expect(result.score).toBe('9-10'); + + // Verify _internal structure (GradeLevelAppropriateness) + expect(result._internal).toHaveProperty('grade'); + expect(result._internal).toHaveProperty('alternative_grade'); + expect(result._internal).toHaveProperty('scaffolding_needed'); + expect(result._internal).toHaveProperty('reasoning'); + + // Verify metadata structure + expect(result.metadata).toHaveProperty('promptVersion'); + expect(result.metadata).toHaveProperty('model'); + expect(result.metadata).toHaveProperty('timestamp'); + expect(result.metadata).toHaveProperty('processingTimeMs'); + + // Verify metadata values + expect(result.metadata.promptVersion).toBe('1.2.0'); + expect(result.metadata.model).toBe('google:gemini-2.5-pro'); + expect(result.metadata.timestamp).toBeInstanceOf(Date); + expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); + + // Verify _internal values + expect(result._internal!.grade).toBe('9-10'); + expect(result._internal!.alternative_grade).toBe('6-8'); + expect(result._internal!.scaffolding_needed).toBeTruthy(); + }); + }); +}); From 95923497751d21fc60033b877e42eec6fd22a3ed Mon Sep 17 00:00:00 2001 From: Adnan Hussain <61515575+adnanrhussain@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:07:15 -0800 Subject: [PATCH 6/9] feat: Add TextComplexityEvaluator and metadata-driven evaluator system (#15) * feat: Add TextComplexityEvaluator and metadata-driven evaluator system --- sdks/typescript/README.md | 66 +++- sdks/typescript/package-lock.json | 48 ++- sdks/typescript/package.json | 1 + sdks/typescript/src/evaluators/base.ts | 89 ++++- .../evaluators/grade-level-appropriateness.ts | 35 +- sdks/typescript/src/evaluators/index.ts | 17 +- .../src/evaluators/sentence-structure.ts | 44 +-- .../src/evaluators/text-complexity.ts | 250 +++++++++++++ sdks/typescript/src/evaluators/vocabulary.ts | 49 +-- sdks/typescript/src/index.ts | 8 +- .../prompts/sentence-structure/complexity.ts | 7 +- .../sentence-structure.integration.test.ts | 16 +- .../evaluators/sentence-structure.test.ts | 4 +- .../unit/evaluators/text-complexity.test.ts | 332 ++++++++++++++++++ .../tests/unit/evaluators/vocabulary.test.ts | 4 +- 15 files changed, 838 insertions(+), 132 deletions(-) create mode 100644 sdks/typescript/src/evaluators/text-complexity.ts create mode 100644 sdks/typescript/tests/unit/evaluators/text-complexity.test.ts diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md index 3965e82..9eec627 100644 --- a/sdks/typescript/README.md +++ b/sdks/typescript/README.md @@ -45,8 +45,8 @@ Evaluates vocabulary complexity using the Qual Text Complexity rubric (SAP). **Constructor:** ```typescript const evaluator = new VocabularyEvaluator({ - googleApiKey: string; // Required - Google API key - openaiApiKey: string; // Required - OpenAI API key + googleApiKey?: string; // Google API key (required by this evaluator) + openaiApiKey?: string; // OpenAI API key (required by this evaluator) maxRetries?: number; // Optional - Max retry attempts (default: 2) telemetry?: boolean | TelemetryOptions; // Optional (default: true) logger?: Logger; // Optional - Custom logger @@ -80,14 +80,14 @@ await evaluator.evaluate(text: string, grade: string) Evaluates sentence structure complexity based on grammatical features. -**Supported Grades:** K-12 +**Supported Grades:** 3-12 **Uses:** OpenAI GPT-4o **Constructor:** ```typescript const evaluator = new SentenceStructureEvaluator({ - openaiApiKey: string; // Required - OpenAI API key + openaiApiKey?: string; // OpenAI API key (required by this evaluator) maxRetries?: number; // Optional - Max retry attempts (default: 2) telemetry?: boolean | TelemetryOptions; // Optional (default: true) logger?: Logger; // Optional - Custom logger @@ -121,7 +121,51 @@ await evaluator.evaluate(text: string, grade: string) --- -### 3. Grade Level Appropriateness Evaluator +### 3. Text Complexity Evaluator + +Composite evaluator that analyzes both vocabulary and sentence structure complexity in parallel. + +**Supported Grades:** 3-12 + +**Uses:** Google Gemini 2.5 Pro + OpenAI GPT-4o (composite) + +**Constructor:** +```typescript +const evaluator = new TextComplexityEvaluator({ + googleApiKey?: string; // Google API key (required by this evaluator) + openaiApiKey?: string; // OpenAI API key (required by this evaluator) + maxRetries?: number; // Optional - Max retry attempts (default: 2) + telemetry?: boolean | TelemetryOptions; // Optional (default: true) + logger?: Logger; // Optional - Custom logger + logLevel?: LogLevel; // Optional - Logging verbosity (default: WARN) +}); +``` + +**API:** +```typescript +await evaluator.evaluate(text: string, grade: string) +``` + +**Returns:** +```typescript +{ + score: { + overall: string; // Overall complexity (highest of the two) + vocabulary: string; // Vocabulary complexity score + sentenceStructure: string; // Sentence structure complexity score + }; + reasoning: string; // Combined reasoning from both evaluators + metadata: EvaluationMetadata; + _internal: { + vocabulary: EvaluationResult | { error: Error }; + sentenceStructure: EvaluationResult | { error: Error }; + }; +} +``` + +--- + +### 4. Grade Level Appropriateness Evaluator Determines appropriate grade level for text. @@ -132,7 +176,7 @@ Determines appropriate grade level for text. **Constructor:** ```typescript const evaluator = new GradeLevelAppropriatenessEvaluator({ - googleApiKey: string; // Required - Google API key + googleApiKey?: string; // Google API key (required by this evaluator) maxRetries?: number; // Optional - Max retry attempts (default: 2) telemetry?: boolean | TelemetryOptions; // Optional (default: true) logger?: Logger; // Optional - Custom logger @@ -253,10 +297,12 @@ See [docs/telemetry.md](./docs/telemetry.md) for telemetry configuration and pri ## Configuration Options -All evaluators support these common options: +All evaluators use the same `BaseEvaluatorConfig` interface: ```typescript interface BaseEvaluatorConfig { + googleApiKey?: string; // Google API key (required by some evaluators) + openaiApiKey?: string; // OpenAI API key (required by some evaluators) maxRetries?: number; // Max API retry attempts (default: 2) telemetry?: boolean | TelemetryOptions; // Telemetry config (default: true) logger?: Logger; // Custom logger (optional) @@ -265,6 +311,12 @@ interface BaseEvaluatorConfig { } ``` +**Note:** Which API keys are required depends on the evaluator. The SDK validates required keys at runtime based on the evaluator's metadata: +- **Vocabulary**: Requires both `googleApiKey` and `openaiApiKey` +- **Sentence Structure**: Requires `openaiApiKey` only +- **Text Complexity**: Requires both `googleApiKey` and `openaiApiKey` +- **Grade Level Appropriateness**: Requires `googleApiKey` only + --- ## License diff --git a/sdks/typescript/package-lock.json b/sdks/typescript/package-lock.json index e3c2e86..d8815a0 100644 --- a/sdks/typescript/package-lock.json +++ b/sdks/typescript/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "compromise": "^14.13.0", + "p-limit": "^5.0.0", "syllable": "^5.0.1", "zod": "^3.22.4" }, @@ -2829,15 +2830,14 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", + "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.0.0" }, "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2858,6 +2858,33 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -3838,12 +3865,11 @@ "dev": true }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json index b269046..d293235 100644 --- a/sdks/typescript/package.json +++ b/sdks/typescript/package.json @@ -60,6 +60,7 @@ }, "dependencies": { "compromise": "^14.13.0", + "p-limit": "^5.0.0", "syllable": "^5.0.1", "zod": "^3.22.4" }, diff --git a/sdks/typescript/src/evaluators/base.ts b/sdks/typescript/src/evaluators/base.ts index bef2ec4..a910100 100644 --- a/sdks/typescript/src/evaluators/base.ts +++ b/sdks/typescript/src/evaluators/base.ts @@ -5,7 +5,7 @@ import { type TelemetryMetadata, type TokenUsage, } from '../telemetry/index.js'; -import { ValidationError } from '../errors.js'; +import { ConfigurationError, ValidationError } from '../errors.js'; import { createLogger, LogLevel, type Logger } from '../logger.js'; /** @@ -80,6 +80,25 @@ export interface BaseEvaluatorConfig { logLevel?: LogLevel; } +/** + * Evaluator metadata interface + * Each evaluator must provide this metadata as static properties + */ +export interface EvaluatorMetadata { + /** Unique identifier for the evaluator (e.g., 'vocabulary', 'sentence-structure') */ + readonly id: string; + /** Human-readable name (e.g., 'Vocabulary', 'Sentence Structure') */ + readonly name: string; + /** Brief description of what the evaluator does */ + readonly description: string; + /** Supported grade levels (e.g., ['3', '4', '5', ...]) */ + readonly supportedGrades: readonly string[]; + /** Whether this evaluator requires a Google API key */ + readonly requiresGoogleKey: boolean; + /** Whether this evaluator requires an OpenAI API key */ + readonly requiresOpenAIKey: boolean; +} + /** * Abstract base class for all evaluators * @@ -88,6 +107,9 @@ export interface BaseEvaluatorConfig { * - Text validation * - Grade validation (with overridable default) * - Metadata creation + * + * Concrete evaluators must implement: + * - static metadata: Provide evaluator metadata (see EvaluatorMetadata interface) */ export abstract class BaseEvaluator { protected telemetryClient?: TelemetryClient; @@ -96,9 +118,34 @@ export abstract class BaseEvaluator { telemetry: Required; }; + /** + * Static metadata for the evaluator + * + * Concrete evaluators MUST define this property. + * + * @example + * ```typescript + * class MyEvaluator extends BaseEvaluator { + * static readonly metadata = { + * id: 'my-evaluator', + * name: 'My Evaluator', + * description: 'Does something useful', + * supportedGrades: ['3', '4', '5'], + * requiresGoogleKey: true, + * requiresOpenAIKey: false, + * }; + * } + * ``` + */ + static readonly metadata: EvaluatorMetadata; + constructor(config: BaseEvaluatorConfig) { // Initialize logger this.logger = createLogger(config.logger, config.logLevel ?? LogLevel.WARN); + + // Validate required API keys based on metadata + this.validateApiKeys(config); + // Normalize telemetry config const telemetryConfig = this.normalizeTelemetryConfig(config.telemetry); @@ -120,6 +167,38 @@ export abstract class BaseEvaluator { } } + /** + * Get metadata for this evaluator instance + * @throws {ConfigurationError} If the subclass has not defined static metadata + */ + protected get metadata(): EvaluatorMetadata { + const meta = (this.constructor as typeof BaseEvaluator).metadata; + if (!meta) { + throw new ConfigurationError( + `${this.constructor.name} must define a static readonly metadata block.` + ); + } + return meta; + } + + /** + * Validate that required API keys are provided based on metadata + * @throws {ConfigurationError} If required API keys are missing + */ + private validateApiKeys(config: BaseEvaluatorConfig): void { + if (this.metadata.requiresGoogleKey && !config.googleApiKey) { + throw new ConfigurationError( + `Google API key is required for ${this.metadata.name} evaluator. Pass googleApiKey in config.` + ); + } + + if (this.metadata.requiresOpenAIKey && !config.openaiApiKey) { + throw new ConfigurationError( + `OpenAI API key is required for ${this.metadata.name} evaluator. Pass openaiApiKey in config.` + ); + } + } + /** * Normalize telemetry config to standard format */ @@ -149,10 +228,12 @@ export abstract class BaseEvaluator { } /** - * Get the evaluator type identifier (e.g., "vocabulary", "sentence-structure") - * Must be implemented by concrete evaluators + * Get the evaluator type identifier from metadata + * @returns The evaluator type ID (e.g., "vocabulary", "sentence-structure") */ - protected abstract getEvaluatorType(): string; + protected getEvaluatorType(): string { + return this.metadata.id; + } /** * Validate text meets requirements diff --git a/sdks/typescript/src/evaluators/grade-level-appropriateness.ts b/sdks/typescript/src/evaluators/grade-level-appropriateness.ts index 525e162..3f43a61 100644 --- a/sdks/typescript/src/evaluators/grade-level-appropriateness.ts +++ b/sdks/typescript/src/evaluators/grade-level-appropriateness.ts @@ -7,15 +7,7 @@ import { import { getSystemPrompt, getUserPrompt } from '../prompts/grade-level-appropriateness/index.js'; import type { EvaluationResult } from '../schemas/index.js'; import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; -import { ConfigurationError, ValidationError, wrapProviderError } from '../errors.js'; - -/** - * Configuration for GradeLevelAppropriatenessEvaluator - */ -export interface GradeLevelAppropriatenessEvaluatorConfig extends BaseEvaluatorConfig { - /** Google API key for grade level evaluation (uses Gemini 2.5 Pro) */ - googleApiKey: string; -} +import { ValidationError, wrapProviderError } from '../errors.js'; /** * Grade Level Appropriateness Evaluator @@ -45,17 +37,21 @@ export interface GradeLevelAppropriatenessEvaluatorConfig extends BaseEvaluatorC * ``` */ export class GradeLevelAppropriatenessEvaluator extends BaseEvaluator { + static readonly metadata = { + id: 'grade-level-appropriateness', + name: 'Grade Level Appropriateness', + description: 'Determines appropriate grade level for text with scaffolding recommendations', + supportedGrades: [] as const, // No grade parameter required - evaluates what grade the text is appropriate for + requiresGoogleKey: true, + requiresOpenAIKey: false, + }; + private provider: LLMProvider; - constructor(config: GradeLevelAppropriatenessEvaluatorConfig) { - // Call base constructor for common setup (telemetry, etc.) + constructor(config: BaseEvaluatorConfig) { + // Call base constructor for common setup (telemetry, API key validation, etc.) super(config); - // Validate required API keys - if (!config.googleApiKey) { - throw new ConfigurationError('Google API key is required. Pass googleApiKey in config.'); - } - // Create Google Gemini provider this.provider = createProvider({ type: 'google', @@ -65,11 +61,6 @@ export class GradeLevelAppropriatenessEvaluator extends BaseEvaluator { }); } - // Implement abstract methods from BaseEvaluator - protected getEvaluatorType(): string { - return 'grade-level-appropriateness'; - } - /** * Evaluate grade level appropriateness for a given text * @@ -194,7 +185,7 @@ export class GradeLevelAppropriatenessEvaluator extends BaseEvaluator { */ export async function evaluateGradeLevelAppropriateness( text: string, - config: GradeLevelAppropriatenessEvaluatorConfig + config: BaseEvaluatorConfig ): Promise> { const evaluator = new GradeLevelAppropriatenessEvaluator(config); return evaluator.evaluate(text); diff --git a/sdks/typescript/src/evaluators/index.ts b/sdks/typescript/src/evaluators/index.ts index 12b55d6..3a42ba2 100644 --- a/sdks/typescript/src/evaluators/index.ts +++ b/sdks/typescript/src/evaluators/index.ts @@ -1,19 +1,28 @@ -export { BaseEvaluator, type BaseEvaluatorConfig, type TelemetryOptions } from './base.js'; +export { + BaseEvaluator, + type BaseEvaluatorConfig, + type TelemetryOptions, + type EvaluatorMetadata, +} from './base.js'; export { VocabularyEvaluator, evaluateVocabulary, - type VocabularyEvaluatorConfig, } from './vocabulary.js'; export { SentenceStructureEvaluator, evaluateSentenceStructure, - type SentenceStructureEvaluatorConfig, } from './sentence-structure.js'; export { GradeLevelAppropriatenessEvaluator, evaluateGradeLevelAppropriateness, - type GradeLevelAppropriatenessEvaluatorConfig, } from './grade-level-appropriateness.js'; + +export { + TextComplexityEvaluator, + evaluateTextComplexity, + type TextComplexityScore, + type TextComplexityInternal, +} from './text-complexity.js'; diff --git a/sdks/typescript/src/evaluators/sentence-structure.ts b/sdks/typescript/src/evaluators/sentence-structure.ts index 67597f4..c1b66d2 100644 --- a/sdks/typescript/src/evaluators/sentence-structure.ts +++ b/sdks/typescript/src/evaluators/sentence-structure.ts @@ -17,12 +17,7 @@ import { import type { EvaluationResult, ComplexityLevel } from '../schemas/index.js'; import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; import type { StageDetail } from '../telemetry/index.js'; -import { ConfigurationError, ValidationError, wrapProviderError } from '../errors.js'; - -/** - * Valid grade levels (K-12) - */ -const VALID_GRADES = new Set(['K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']); +import { ValidationError, wrapProviderError } from '../errors.js'; /** * Internal data structure for sentence structure evaluation @@ -54,14 +49,6 @@ function normalizeLabel(label: string | null | undefined): string | null { return mapping[normalized] || null; // Return null if no mapping found } -/** - * Configuration for SentenceStructureEvaluator - */ -export interface SentenceStructureEvaluatorConfig extends BaseEvaluatorConfig { - /** OpenAI API key for sentence analysis and complexity evaluation (uses GPT-4o) */ - openaiApiKey: string; -} - /** * Sentence Structure Evaluator * @@ -88,18 +75,22 @@ export interface SentenceStructureEvaluatorConfig extends BaseEvaluatorConfig { * ``` */ export class SentenceStructureEvaluator extends BaseEvaluator { + static readonly metadata = { + id: 'sentence-structure', + name: 'Sentence Structure', + description: 'Evaluates sentence structure complexity based on grammatical features', + supportedGrades: ['3', '4', '5', '6', '7', '8', '9', '10', '11', '12'] as const, + requiresGoogleKey: false, + requiresOpenAIKey: true, + }; + private analysisProvider: LLMProvider; private complexityProvider: LLMProvider; - constructor(config: SentenceStructureEvaluatorConfig) { - // Call base constructor for common setup (telemetry, etc.) + constructor(config: BaseEvaluatorConfig) { + // Call base constructor for common setup (telemetry, API key validation, etc.) super(config); - // Validate required API keys - if (!config.openaiApiKey) { - throw new ConfigurationError('OpenAI API key is required. Pass openaiApiKey in config.'); - } - // Create OpenAI GPT-4o provider for both stages this.analysisProvider = createProvider({ type: 'openai', @@ -116,16 +107,11 @@ export class SentenceStructureEvaluator extends BaseEvaluator { }); } - // Implement abstract methods from BaseEvaluator - protected getEvaluatorType(): string { - return 'sentence-structure'; - } - /** * Evaluate sentence structure complexity for a given text and grade level * * @param text - The text to evaluate - * @param grade - The target grade level (K-12) + * @param grade - The target grade level (3-12) * @returns Evaluation result with complexity score and detailed analysis * @throws {ValidationError} If text is empty, too short/long, or grade is invalid * @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError) @@ -147,7 +133,7 @@ export class SentenceStructureEvaluator extends BaseEvaluator { try { // Validate inputs — inside try so validation errors are telemetered. this.validateText(text); - this.validateGrade(grade, VALID_GRADES); + this.validateGrade(grade, new Set(SentenceStructureEvaluator.metadata.supportedGrades)); this.logger.debug('Stage 1: Analyzing sentence structure', { evaluator: 'sentence-structure', operation: 'sentence_analysis', @@ -377,7 +363,7 @@ export class SentenceStructureEvaluator extends BaseEvaluator { export async function evaluateSentenceStructure( text: string, grade: string, - config: SentenceStructureEvaluatorConfig + config: BaseEvaluatorConfig ): Promise> { const evaluator = new SentenceStructureEvaluator(config); return evaluator.evaluate(text, grade); diff --git a/sdks/typescript/src/evaluators/text-complexity.ts b/sdks/typescript/src/evaluators/text-complexity.ts new file mode 100644 index 0000000..5493a72 --- /dev/null +++ b/sdks/typescript/src/evaluators/text-complexity.ts @@ -0,0 +1,250 @@ +import pLimit from 'p-limit'; +import { VocabularyEvaluator } from './vocabulary.js'; +import { SentenceStructureEvaluator } from './sentence-structure.js'; +import type { BaseEvaluatorConfig } from './base.js'; +import { BaseEvaluator } from './base.js'; +import type { EvaluationResult } from '../schemas/index.js'; + +/** + * Internal data structure for text complexity evaluation + * Stores either successful evaluation results or errors from sub-evaluators + */ +export interface TextComplexityInternal { + vocabulary: EvaluationResult | { error: Error }; + sentenceStructure: EvaluationResult | { error: Error }; +} + +/** + * Composite score for text complexity + */ +export interface TextComplexityScore { + /** Vocabulary complexity score */ + vocabulary: string; + /** Sentence structure complexity score */ + sentenceStructure: string; +} + +/** + * Text Complexity Evaluator + * + * Composite evaluator that analyzes both vocabulary and sentence structure complexity. + * Runs both evaluations in parallel with concurrency control to avoid rate limiting. + * + * Uses: + * - VocabularyEvaluator (Google Gemini 2.5 Pro + OpenAI GPT-4o) + * - SentenceStructureEvaluator (OpenAI GPT-4o) + * + * @example + * ```typescript + * const evaluator = new TextComplexityEvaluator({ + * googleApiKey: process.env.GOOGLE_API_KEY, + * openaiApiKey: process.env.OPENAI_API_KEY + * }); + * + * const result = await evaluator.evaluate(text, "5"); + * console.log(result.score.vocabulary); + * console.log(result.score.sentenceStructure); + * ``` + */ +export class TextComplexityEvaluator extends BaseEvaluator { + static readonly metadata = { + id: 'text-complexity', + name: 'Text Complexity', + description: 'Composite evaluator analyzing vocabulary and sentence structure complexity', + supportedGrades: ['3', '4', '5', '6', '7', '8', '9', '10', '11', '12'] as const, + requiresGoogleKey: true, + requiresOpenAIKey: true, + }; + + private vocabularyEvaluator: VocabularyEvaluator; + private sentenceStructureEvaluator: SentenceStructureEvaluator; + private limit: ReturnType; + + constructor(config: BaseEvaluatorConfig) { + // Call base constructor for common setup (telemetry, API key validation, etc.) + super(config); + + // Create child evaluators with same config + this.vocabularyEvaluator = new VocabularyEvaluator(config); + this.sentenceStructureEvaluator = new SentenceStructureEvaluator(config); + + // Create concurrency limiter (max 3 concurrent operations) + this.limit = pLimit(3); + } + + /** + * Evaluate text complexity for a given text and grade level + * + * Runs vocabulary and sentence structure evaluations in parallel with concurrency control. + * + * @param text - The text to evaluate + * @param grade - The target grade level (3-12) + * @returns Evaluation result with composite complexity score + * @throws {Error} If text is empty or grade is invalid + */ + async evaluate( + text: string, + grade: string + ): Promise> { + this.logger.info('Starting text complexity evaluation', { + evaluator: 'text-complexity', + operation: 'evaluate', + grade, + textLength: text.length, + }); + + // Use inherited validation methods + this.validateText(text); + this.validateGrade(grade, new Set(TextComplexityEvaluator.metadata.supportedGrades)); + + const startTime = Date.now(); + + // Run both evaluators in parallel with concurrency control + const [vocabResult, sentenceResult] = await Promise.all([ + this.limit(() => this.runSubEvaluator(this.vocabularyEvaluator, text, grade)), + this.limit(() => this.runSubEvaluator(this.sentenceStructureEvaluator, text, grade)), + ]); + + const latencyMs = Date.now() - startTime; + + // Build combined reasoning + const reasoning = this.buildCombinedReasoning(vocabResult, sentenceResult); + + // Check if any evaluations failed + const vocabFailed = 'error' in vocabResult; + const sentenceFailed = 'error' in sentenceResult; + const hasFailures = vocabFailed || sentenceFailed; + + if (hasFailures) { + const errors: string[] = []; + if (vocabFailed) { + errors.push(`Vocabulary evaluation failed: ${vocabResult.error.message}`); + } + if (sentenceFailed) { + errors.push(`Sentence structure evaluation failed: ${sentenceResult.error.message}`); + } + + this.logger.error('Text complexity evaluation completed with errors', { + evaluator: 'text-complexity', + operation: 'evaluate', + grade, + errors, + processingTimeMs: latencyMs, + }); + + // If both failed, throw error + if (vocabFailed && sentenceFailed) { + throw new Error( + `Text complexity evaluation failed: ${errors.join('; ')}` + ); + } + } + + const result = { + score: { + vocabulary: vocabFailed ? 'N/A' : vocabResult.score, + sentenceStructure: sentenceFailed ? 'N/A' : sentenceResult.score, + }, + reasoning, + metadata: { + promptVersion: '1.0', + model: 'composite:gemini-2.5-pro+gpt-4o', + timestamp: new Date(), + processingTimeMs: latencyMs, + }, + _internal: { + vocabulary: vocabResult, + sentenceStructure: sentenceResult, + }, + }; + + // Send telemetry (fire-and-forget) + this.sendTelemetry({ + status: hasFailures ? 'error' : 'success', + latencyMs, + textLength: text.length, + grade, + provider: 'composite:google+openai', + errorCode: hasFailures ? 'PartialFailure' : undefined, + inputText: text, + }).catch(() => { + // Ignore telemetry errors + }); + + this.logger.info('Text complexity evaluation completed', { + evaluator: 'text-complexity', + operation: 'evaluate', + grade, + processingTimeMs: latencyMs, + hasFailures, + }); + + return result; + } + + /** + * Run a sub-evaluator with error handling + * Returns the evaluation result or an error object + */ + private async runSubEvaluator( + evaluator: { evaluate(text: string, grade: string): Promise> }, + text: string, + grade: string + ): Promise | { error: Error }> { + try { + return await evaluator.evaluate(text, grade); + } catch (error) { + return { + error: error instanceof Error ? error : new Error(String(error)), + }; + } + } + + /** + * Build combined reasoning from individual results + */ + private buildCombinedReasoning( + vocabResult: EvaluationResult | { error: Error }, + sentenceResult: EvaluationResult | { error: Error } + ): string { + const parts: string[] = []; + + if ('error' in vocabResult) { + parts.push(`Vocabulary Complexity: Evaluation failed - ${vocabResult.error.message}`); + } else { + parts.push(`Vocabulary Complexity (${vocabResult.score}):\n${vocabResult.reasoning}`); + } + + if ('error' in sentenceResult) { + parts.push(`Sentence Structure Complexity: Evaluation failed - ${sentenceResult.error.message}`); + } else { + parts.push(`Sentence Structure Complexity (${sentenceResult.score}):\n${sentenceResult.reasoning}`); + } + + return parts.join('\n\n'); + } +} + +/** + * Functional API for text complexity evaluation + * + * @example + * ```typescript + * const result = await evaluateTextComplexity( + * "The cat sat on the mat.", + * "5", + * { + * googleApiKey: process.env.GOOGLE_API_KEY, + * openaiApiKey: process.env.OPENAI_API_KEY + * } + * ); + * ``` + */ +export async function evaluateTextComplexity( + text: string, + grade: string, + config: BaseEvaluatorConfig +): Promise> { + const evaluator = new TextComplexityEvaluator(config); + return evaluator.evaluate(text, grade); +} diff --git a/sdks/typescript/src/evaluators/vocabulary.ts b/sdks/typescript/src/evaluators/vocabulary.ts index 912e8a2..9b145fa 100644 --- a/sdks/typescript/src/evaluators/vocabulary.ts +++ b/sdks/typescript/src/evaluators/vocabulary.ts @@ -14,23 +14,7 @@ import { import type { EvaluationResult } from '../schemas/index.js'; import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; import type { StageDetail } from '../telemetry/index.js'; -import { ConfigurationError, ValidationError, wrapProviderError } from '../errors.js'; - -/** - * Valid grade levels (3-12) - */ -const VALID_GRADES = new Set(['3', '4', '5', '6', '7', '8', '9', '10', '11', '12']); - -/** - * Configuration for VocabularyEvaluator - */ -export interface VocabularyEvaluatorConfig extends BaseEvaluatorConfig { - /** Google API key for complexity evaluation (uses Gemini 2.5 Pro) */ - googleApiKey: string; - - /** OpenAI API key for background knowledge generation (uses GPT-4o) */ - openaiApiKey: string; -} +import { ValidationError, wrapProviderError } from '../errors.js'; /** * Vocabulary Evaluator @@ -59,23 +43,23 @@ export interface VocabularyEvaluatorConfig extends BaseEvaluatorConfig { * ``` */ export class VocabularyEvaluator extends BaseEvaluator { + static readonly metadata = { + id: 'vocabulary', + name: 'Vocabulary', + description: 'Evaluates vocabulary complexity of educational texts relative to grade level', + supportedGrades: ['3', '4', '5', '6', '7', '8', '9', '10', '11', '12'] as const, + requiresGoogleKey: true, + requiresOpenAIKey: true, + }; + private grades34ComplexityProvider: LLMProvider; private otherGradesComplexityProvider: LLMProvider; private backgroundKnowledgeProvider: LLMProvider; - constructor(config: VocabularyEvaluatorConfig) { - // Call base constructor for common setup (telemetry, etc.) + constructor(config: BaseEvaluatorConfig) { + // Call base constructor for common setup (telemetry, API key validation, etc.) super(config); - // Validate required API keys - if (!config.googleApiKey) { - throw new ConfigurationError('Google API key is required. Pass googleApiKey in config.'); - } - - if (!config.openaiApiKey) { - throw new ConfigurationError('OpenAI API key is required. Pass openaiApiKey in config.'); - } - // Create Google Gemini provider for complexity evaluation (grades 3-4) this.grades34ComplexityProvider = createProvider({ type: 'google', @@ -101,11 +85,6 @@ export class VocabularyEvaluator extends BaseEvaluator { }); } - // Implement abstract methods from BaseEvaluator - protected getEvaluatorType(): string { - return 'vocabulary'; - } - /** * Evaluate vocabulary complexity for a given text and grade level * @@ -136,7 +115,7 @@ export class VocabularyEvaluator extends BaseEvaluator { // Validate inputs — inside try so validation errors are telemetered. // If partners consistently pass invalid grades/text, telemetry will surface documentation gaps. this.validateText(text); - this.validateGrade(grade, VALID_GRADES); + this.validateGrade(grade, new Set(VocabularyEvaluator.metadata.supportedGrades)); this.logger.debug('Stage 1: Generating background knowledge', { evaluator: 'vocabulary', operation: 'background_knowledge', @@ -346,7 +325,7 @@ export class VocabularyEvaluator extends BaseEvaluator { export async function evaluateVocabulary( text: string, grade: string, - config: VocabularyEvaluatorConfig + config: BaseEvaluatorConfig ): Promise> { const evaluator = new VocabularyEvaluator(config); return evaluator.evaluate(text, grade); diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index a84fcc0..93da1d0 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -62,15 +62,17 @@ export { GradeLevelAppropriatenessSchema } from './schemas/grade-level-appropria export { VocabularyEvaluator, evaluateVocabulary, - type VocabularyEvaluatorConfig, SentenceStructureEvaluator, evaluateSentenceStructure, - type SentenceStructureEvaluatorConfig, GradeLevelAppropriatenessEvaluator, evaluateGradeLevelAppropriateness, - type GradeLevelAppropriatenessEvaluatorConfig, + TextComplexityEvaluator, + evaluateTextComplexity, + type TextComplexityScore, + type TextComplexityInternal, type BaseEvaluatorConfig, type TelemetryOptions, + type EvaluatorMetadata, } from './evaluators/index.js'; // Features diff --git a/sdks/typescript/src/prompts/sentence-structure/complexity.ts b/sdks/typescript/src/prompts/sentence-structure/complexity.ts index 32189ea..fd90505 100644 --- a/sdks/typescript/src/prompts/sentence-structure/complexity.ts +++ b/sdks/typescript/src/prompts/sentence-structure/complexity.ts @@ -14,7 +14,7 @@ export function getSystemPromptComplexity(): string { /** * Get the appropriate rubric based on grade level - * @param grade - The target grade level (K-12) + * @param grade - The target grade level (3-12) * @returns The rubric text for the grade level */ export function getRubricForGrade(grade: string): string { @@ -22,11 +22,8 @@ export function getRubricForGrade(grade: string): string { return RUBRIC_GRADE_3; } else if (grade === '4') { return RUBRIC_GRADE_4; - } else if (['5', '6', '7', '8', '9', '10', '11', '12'].includes(grade)) { - return RUBRIC_GRADES_5_12; } else { - // K, 1, 2 - no specific rubric, use general principles - return 'No specific rubric available for this grade. Use general linguistic principles.'; + return RUBRIC_GRADES_5_12; } } diff --git a/sdks/typescript/tests/integration/sentence-structure.integration.test.ts b/sdks/typescript/tests/integration/sentence-structure.integration.test.ts index dc802d4..a9c9bd5 100644 --- a/sdks/typescript/tests/integration/sentence-structure.integration.test.ts +++ b/sdks/typescript/tests/integration/sentence-structure.integration.test.ts @@ -8,7 +8,7 @@ import { /** * Sentence Structure Evaluator Integration Tests * - * Test cases cover grades 2-6 with varying complexity levels. + * Test cases cover grades 3-6 with varying complexity levels. * * Each test uses a retry mechanism (up to 3 attempts) to account for LLM non-determinism, * with short-circuiting on first expected match. If no expected match is found after all @@ -29,13 +29,13 @@ const describeIntegration = SKIP_INTEGRATION ? describe.skip : describe; const TEST_TIMEOUT_MS = 2 * 60 * 1000; const TEST_CASES: BaseTestCase[] = [ - { - id: 'SS2', - grade: '2', - text: "The Roman Empire was a powerful empire that lasted for hundreds of years. It started as a small village in Italy and grew into a huge empire that controlled much of Europe, Asia, and Africa. The Roman Empire had many strong leaders like Julius Caesar and Augustus. These leaders helped the empire grow and become very powerful.\n \n\n The Roman Empire had a period of peace and prosperity called the Pax Romana. This time was good for the empire, but it didn't last forever. The empire started to have problems. The army became weaker, and the economy had problems. The empire was also attacked by groups of people called barbarians.\n \n\n The Roman Empire was divided into two parts: the Western Roman Empire and the Eastern Roman Empire. The Western Roman Empire eventually fell apart in 476 AD. The Eastern Roman Empire, also known as the Byzantine Empire, lasted for many more years. The Roman Empire left behind many things that we still use today, like the Roman alphabet and the calendar.", - expected: 'moderately complex', - acceptable: ['slightly complex', 'very complex'], - }, + // { + // id: 'SS2', + // grade: '2', + // text: "The Roman Empire was a powerful empire that lasted for hundreds of years. It started as a small village in Italy and grew into a huge empire that controlled much of Europe, Asia, and Africa. The Roman Empire had many strong leaders like Julius Caesar and Augustus. These leaders helped the empire grow and become very powerful.\n \n\n The Roman Empire had a period of peace and prosperity called the Pax Romana. This time was good for the empire, but it didn't last forever. The empire started to have problems. The army became weaker, and the economy had problems. The empire was also attacked by groups of people called barbarians.\n \n\n The Roman Empire was divided into two parts: the Western Roman Empire and the Eastern Roman Empire. The Western Roman Empire eventually fell apart in 476 AD. The Eastern Roman Empire, also known as the Byzantine Empire, lasted for many more years. The Roman Empire left behind many things that we still use today, like the Roman alphabet and the calendar.", + // expected: 'moderately complex', + // acceptable: ['slightly complex', 'very complex'], + // }, { id: 'SS3', grade: '3', diff --git a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts index 96afdc5..a4971e2 100644 --- a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts +++ b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts @@ -112,7 +112,7 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { describe('Successful Evaluation Flow', () => { it('should successfully evaluate text through both stages', async () => { const testText = 'The cat sat on the mat. It was sleeping peacefully.'; - const testGrade = 'K'; + const testGrade = '3'; // Mock sentence analysis response vi.mocked(mockAnalysisProvider.generateStructured).mockResolvedValue({ @@ -129,7 +129,7 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ data: { answer: 'Slightly Complex', - reasoning: 'The text uses simple sentence structures appropriate for kindergarten.', + reasoning: 'The text uses simple sentence structures appropriate for third grade.', }, model: 'gpt-4o', usage: { diff --git a/sdks/typescript/tests/unit/evaluators/text-complexity.test.ts b/sdks/typescript/tests/unit/evaluators/text-complexity.test.ts new file mode 100644 index 0000000..a3c48bf --- /dev/null +++ b/sdks/typescript/tests/unit/evaluators/text-complexity.test.ts @@ -0,0 +1,332 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { TextComplexityEvaluator } from '../../../src/evaluators/text-complexity.js'; +import { ConfigurationError, ValidationError } from '../../../src/errors.js'; + +// Mock telemetry to avoid real HTTP calls +vi.mock('../../../src/telemetry/client.js', () => ({ + TelemetryClient: class MockTelemetryClient { + send = vi.fn().mockResolvedValue(undefined); + }, +})); + +// Mock providers to avoid real API calls +vi.mock('../../../src/providers/index.js', () => ({ + createProvider: vi.fn(() => ({ + generateStructured: vi.fn().mockResolvedValue({ + data: { + complexity_score: 'moderately complex', + reasoning: 'Test reasoning', + answer: 'Moderately Complex', + }, + usage: { inputTokens: 100, outputTokens: 50 }, + latencyMs: 100, + }), + generateText: vi.fn().mockResolvedValue({ + text: 'Test background knowledge', + usage: { inputTokens: 100, outputTokens: 50 }, + latencyMs: 100, + }), + })), +})); + +describe('TextComplexityEvaluator', () => { + describe('Metadata', () => { + it('should have correct metadata', () => { + expect(TextComplexityEvaluator.metadata.id).toBe('text-complexity'); + expect(TextComplexityEvaluator.metadata.name).toBe('Text Complexity'); + expect(TextComplexityEvaluator.metadata.requiresGoogleKey).toBe(true); + expect(TextComplexityEvaluator.metadata.requiresOpenAIKey).toBe(true); + expect(TextComplexityEvaluator.metadata.supportedGrades).toEqual([ + '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', + ]); + }); + }); + + describe('Constructor', () => { + it('should create evaluator with valid config', () => { + const evaluator = new TextComplexityEvaluator({ + googleApiKey: 'test-google-key', + openaiApiKey: 'test-openai-key', + telemetry: false, + }); + + expect(evaluator).toBeDefined(); + }); + + it('should throw error when Google API key is missing', () => { + expect(() => { + new TextComplexityEvaluator({ + openaiApiKey: 'test-openai-key', + telemetry: false, + }); + }).toThrow(ConfigurationError); + expect(() => { + new TextComplexityEvaluator({ + openaiApiKey: 'test-openai-key', + telemetry: false, + }); + }).toThrow('Google API key is required for Text Complexity evaluator'); + }); + + it('should throw error when OpenAI API key is missing', () => { + expect(() => { + new TextComplexityEvaluator({ + googleApiKey: 'test-google-key', + telemetry: false, + }); + }).toThrow(ConfigurationError); + expect(() => { + new TextComplexityEvaluator({ + googleApiKey: 'test-google-key', + telemetry: false, + }); + }).toThrow('OpenAI API key is required for Text Complexity evaluator'); + }); + + it('should throw error when both API keys are missing', () => { + expect(() => { + new TextComplexityEvaluator({ + telemetry: false, + }); + }).toThrow(ConfigurationError); + }); + }); + + describe('evaluate()', () => { + let evaluator: TextComplexityEvaluator; + let vocabSpy: any; + let sentenceSpy: any; + + beforeEach(() => { + evaluator = new TextComplexityEvaluator({ + googleApiKey: 'test-google-key', + openaiApiKey: 'test-openai-key', + telemetry: false, + }); + + // Mock the child evaluators' evaluate methods + vocabSpy = vi.spyOn((evaluator as any).vocabularyEvaluator, 'evaluate').mockResolvedValue({ + score: 'moderately complex', + reasoning: 'Vocabulary test reasoning', + metadata: { + promptVersion: '1.0', + model: 'gemini-2.5-pro + gpt-4o', + timestamp: new Date(), + processingTimeMs: 100, + }, + _internal: {}, + }); + + sentenceSpy = vi.spyOn((evaluator as any).sentenceStructureEvaluator, 'evaluate').mockResolvedValue({ + score: 'Moderately Complex', + reasoning: 'Sentence structure test reasoning', + metadata: { + promptVersion: '1.0', + model: 'gpt-4o', + timestamp: new Date(), + processingTimeMs: 100, + }, + _internal: {}, + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should evaluate text successfully', async () => { + const text = 'The cat sat on the mat.'; + const grade = '5'; + + const result = await evaluator.evaluate(text, grade); + + expect(result).toBeDefined(); + expect(result.score).toBeDefined(); + expect(result.score.vocabulary).toBeDefined(); + expect(result.score.sentenceStructure).toBeDefined(); + expect(result.reasoning).toBeDefined(); + expect(result.metadata).toBeDefined(); + expect(result.metadata.model).toBe('composite:gemini-2.5-pro+gpt-4o'); + expect(result._internal).toBeDefined(); + expect(result._internal!.vocabulary).toBeDefined(); + expect(result._internal!.sentenceStructure).toBeDefined(); + }); + + it('should validate text input', async () => { + await expect(evaluator.evaluate('', '5')).rejects.toThrow(ValidationError); + await expect(evaluator.evaluate(' ', '5')).rejects.toThrow( + 'Text cannot be empty or contain only whitespace' + ); + await expect(evaluator.evaluate('abc', '5')).rejects.toThrow( + 'Text is too short' + ); + }); + + it('should validate grade input', async () => { + const text = 'The cat sat on the mat.'; + + await expect(evaluator.evaluate(text, 'invalid')).rejects.toThrow( + ValidationError + ); + await expect(evaluator.evaluate(text, 'invalid')).rejects.toThrow( + 'Invalid grade "invalid"' + ); + + // Grades outside supported range (K, 1, 2 not supported) + await expect(evaluator.evaluate(text, 'K')).rejects.toThrow(ValidationError); + await expect(evaluator.evaluate(text, '1')).rejects.toThrow(ValidationError); + await expect(evaluator.evaluate(text, '2')).rejects.toThrow(ValidationError); + }); + + it('should accept all supported grades', async () => { + const text = 'The cat sat on the mat.'; + const supportedGrades = ['3', '4', '5', '6', '7', '8', '9', '10', '11', '12']; + + for (const grade of supportedGrades) { + const result = await evaluator.evaluate(text, grade); + expect(result).toBeDefined(); + } + }); + + it('should run both evaluators in parallel', async () => { + const text = 'The cat sat on the mat.'; + const grade = '5'; + + const startTime = Date.now(); + const result = await evaluator.evaluate(text, grade); + const duration = Date.now() - startTime; + + // With mocked providers that take ~100ms each, parallel execution should be faster than sequential + // Sequential would be ~200ms, parallel should be ~100ms + // Allow some overhead but should be significantly less than 200ms + expect(duration).toBeLessThan(200); + + expect('error' in result._internal!.vocabulary).toBe(false); + expect('error' in result._internal!.sentenceStructure).toBe(false); + }); + + it('should handle partial failures gracefully', async () => { + const text = 'The cat sat on the mat.'; + const grade = '5'; + + // Override the spy to make vocabulary fail but sentence structure succeed + vocabSpy.mockRejectedValue(new Error('Vocabulary evaluation failed')); + + const result = await evaluator.evaluate(text, grade); + + expect(result).toBeDefined(); + expect('error' in result._internal!.vocabulary).toBe(true); + expect((result._internal!.vocabulary as { error: Error }).error).toBeDefined(); + expect('error' in result._internal!.sentenceStructure).toBe(false); + expect(result.score.vocabulary).toBe('N/A'); + expect(result.score.sentenceStructure).not.toBe('N/A'); + }); + + it('should throw when both evaluators fail', async () => { + const text = 'The cat sat on the mat.'; + const grade = '5'; + + // Override both spies to fail + vocabSpy.mockRejectedValue(new Error('Vocabulary evaluation failed')); + sentenceSpy.mockRejectedValue(new Error('Sentence structure evaluation failed')); + + await expect(evaluator.evaluate(text, grade)).rejects.toThrow( + 'Text complexity evaluation failed' + ); + }); + + it('should determine overall complexity correctly', async () => { + const text = 'The cat sat on the mat.'; + const grade = '5'; + + // Override vocabulary to return "moderately complex" + vocabSpy.mockResolvedValue({ + score: 'moderately complex', + reasoning: 'Vocab reasoning', + metadata: { + promptVersion: '1.0', + model: 'gemini-2.5-pro', + timestamp: new Date(), + processingTimeMs: 100, + }, + _internal: {}, + }); + + // Override sentence structure to return "Slightly Complex" + sentenceSpy.mockResolvedValue({ + score: 'Slightly Complex', + reasoning: 'Sentence reasoning', + metadata: { + promptVersion: '1.0', + model: 'gpt-4o', + timestamp: new Date(), + processingTimeMs: 100, + }, + _internal: {}, + }); + + const result = await evaluator.evaluate(text, grade); + + expect(result.score.vocabulary).toBe('moderately complex'); + expect(result.score.sentenceStructure).toBe('Slightly Complex'); + }); + + it('should build combined reasoning from both evaluators', async () => { + const text = 'The cat sat on the mat.'; + const grade = '5'; + + // Override both evaluators with specific reasoning + vocabSpy.mockResolvedValue({ + score: 'moderately complex', + reasoning: 'This is the vocabulary reasoning.', + metadata: { + promptVersion: '1.0', + model: 'gemini-2.5-pro', + timestamp: new Date(), + processingTimeMs: 100, + }, + _internal: {}, + }); + + sentenceSpy.mockResolvedValue({ + score: 'Slightly Complex', + reasoning: 'This is the sentence structure reasoning.', + metadata: { + promptVersion: '1.0', + model: 'gpt-4o', + timestamp: new Date(), + processingTimeMs: 100, + }, + _internal: {}, + }); + + const result = await evaluator.evaluate(text, grade); + + expect(result.reasoning).toContain('Vocabulary Complexity'); + expect(result.reasoning).toContain('This is the vocabulary reasoning.'); + expect(result.reasoning).toContain('Sentence Structure Complexity'); + expect(result.reasoning).toContain('This is the sentence structure reasoning.'); + }); + }); + + describe('Concurrency Control', () => { + it('should use p-limit for concurrency control', async () => { + const evaluator = new TextComplexityEvaluator({ + googleApiKey: 'test-google-key', + openaiApiKey: 'test-openai-key', + telemetry: false, + }); + + // Check that limit is defined + expect((evaluator as any).limit).toBeDefined(); + + const text = 'The cat sat on the mat.'; + const grade = '5'; + + await evaluator.evaluate(text, grade); + + // The limit should have been used (both calls go through it) + expect((evaluator as any).limit).toBeDefined(); + }); + }); +}); diff --git a/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts b/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts index 2ce906a..9792acb 100644 --- a/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts +++ b/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts @@ -39,14 +39,14 @@ describe('VocabularyEvaluator - Constructor Validation', () => { expect(() => new VocabularyEvaluator({ googleApiKey: '', openaiApiKey: 'test-openai-key', - })).toThrow('Google API key is required. Pass googleApiKey in config.'); + })).toThrow('Google API key is required for Vocabulary evaluator. Pass googleApiKey in config.'); }); it('should throw error when OpenAI API key is missing', () => { expect(() => new VocabularyEvaluator({ googleApiKey: 'test-google-key', openaiApiKey: '', - })).toThrow('OpenAI API key is required. Pass openaiApiKey in config.'); + })).toThrow('OpenAI API key is required for Vocabulary evaluator. Pass openaiApiKey in config.'); }); }); From 7295f36ec22534e3fbf9db96fdf6bc9893640095 Mon Sep 17 00:00:00 2001 From: Adnan Hussain <61515575+adnanrhussain@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:54:34 -0800 Subject: [PATCH 7/9] chore: Update license info (#18) --- sdks/typescript/LICENSE | 25 +++++++++++++++++-------- sdks/typescript/THIRD_PARTY_LICENSES.md | 3 +++ 2 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 sdks/typescript/THIRD_PARTY_LICENSES.md diff --git a/sdks/typescript/LICENSE b/sdks/typescript/LICENSE index e8c406f..b59af92 100644 --- a/sdks/typescript/LICENSE +++ b/sdks/typescript/LICENSE @@ -1,12 +1,21 @@ -The Evaluator code is licensed under [MIT](https://opensource.org/license/mit). +MIT License -Evaluators content (including the prompt and settings information) is provided by Learning Commons under the CC BY 4.0 International license ([CC BY 4.0](https://creativecommons.org/licenses/by/4.0/deed.en)). +Copyright (c) 2026 Learning Commons -Annotated CLEAR Corpus is provided by Learning Commons (including annotations and enhancements) under CC BY-NC-SA 4.0 ([CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/deed.en)). The original dataset from CLEAR Corpus can be found at [The CLEAR Corpus by CommonLit](https://www.commonlit.org/blog/introducing-the-clear-corpus-an-open-dataset-to-advance-research-28ff8cfea84a/) licensed under CC BY-NC-SA 4.0. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -**How to Cite the Evaluator Code:** -Learning Commons (2025). Evaluators. GitHub. [https://github.com/learning-commons-org/evaluators](https://github.com/learning-commons-org/evaluators). -Licensed under MIT. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -**How to Cite the Evaluator:** -Learning Commons. (2025). Evaluators content (including the prompt and settings information) is available at GitHub. [https://github.com/learning-commons-org/evaluators](https://github.com/learning-commons-org/evaluators) Licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/deed.en) +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/sdks/typescript/THIRD_PARTY_LICENSES.md b/sdks/typescript/THIRD_PARTY_LICENSES.md new file mode 100644 index 0000000..b768123 --- /dev/null +++ b/sdks/typescript/THIRD_PARTY_LICENSES.md @@ -0,0 +1,3 @@ +The Evaluator code and SDK is licensed under [MIT](https://opensource.org/license/mit). + +Evaluators content (including the prompt and settings information) is provided by Learning Commons under the CC BY 4.0 International license ([CC BY 4.0](https://creativecommons.org/licenses/by/4.0/deed.en)). From f21a70451fbbc2ddd0c80fe1c7a11930d45baf65 Mon Sep 17 00:00:00 2001 From: Adnan Hussain <61515575+adnanrhussain@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:58:14 -0700 Subject: [PATCH 8/9] refactor: Unify return types and casing (#19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(types): unify TextComplexityLevel and tighten evaluator return types - Rename ComplexityLevel → TextComplexityLevel; all evaluators now share a single snake_case enum (slightly/moderately/very/exceedingly_complex) - Remove VocabularyComplexityLevel (lowercase spaced) in favour of unified enum; complexity_score now uses snake_case values - Add COMPLEXITY_LEVEL_LABELS map for display strings (e.g. 'Slightly complex') - Rename internal types: VocabularyComplexity → VocabularyInternal, GradeLevelAppropriateness → GradeLevelAppropriatenessInternal - Move SentenceStructureInternal to its schema file and export it - Tighten EvaluationResult TScore: string → TextComplexityLevel (vocab, SS) and string → GradeBand (GLA) - Redesign TextComplexityEvaluator to return TextComplexityResult map instead of a nested EvaluationResult wrapper; each key holds the sub-evaluator result or { error: Error } directly - Remove buildCombinedReasoning; callers access per-evaluator reasoning directly from the result map - Make runSubEvaluator generic to preserve TScore/TInternal types through the p-limit boundary - normalizeLabel normalises LLM output variations to canonical snake_case values; returns TextComplexityLevel | null (no cast needed) - Update unit and integration tests for new enum values and result shape * fix: set correct promptVersion per evaluator based on prompt changelog * refactor: remove unused metadata * refactor: remove unused exports * refactor: remove unused metadata * fix: telemetry endpoint --- sdks/typescript/README.md | 6 - sdks/typescript/src/evaluators/base.ts | 2 +- .../evaluators/grade-level-appropriateness.ts | 10 +- sdks/typescript/src/evaluators/index.ts | 3 +- .../src/evaluators/sentence-structure.ts | 51 +++---- .../src/evaluators/text-complexity.ts | 124 +++++------------- sdks/typescript/src/evaluators/vocabulary.ts | 14 +- sdks/typescript/src/index.ts | 16 +-- .../schemas/grade-level-appropriateness.ts | 2 +- sdks/typescript/src/schemas/index.ts | 7 +- sdks/typescript/src/schemas/outputs.ts | 30 ++--- .../src/schemas/sentence-structure.ts | 13 +- sdks/typescript/src/schemas/vocabulary.ts | 20 +-- .../sentence-structure.integration.test.ts | 24 ++-- .../vocabulary.integration.test.ts | 28 ++-- .../grade-level-appropriateness.test.ts | 4 - .../evaluators/sentence-structure.test.ts | 10 +- .../unit/evaluators/text-complexity.test.ts | 77 +++++------ .../tests/unit/evaluators/vocabulary.test.ts | 12 +- 19 files changed, 161 insertions(+), 292 deletions(-) diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md index 9eec627..51f8726 100644 --- a/sdks/typescript/README.md +++ b/sdks/typescript/README.md @@ -65,9 +65,7 @@ await evaluator.evaluate(text: string, grade: string) score: 'slightly complex' | 'moderately complex' | 'very complex' | 'exceedingly complex'; reasoning: string; metadata: { - promptVersion: string; model: string; - timestamp: Date; processingTimeMs: number; }; _internal: VocabularyComplexity; // Detailed analysis @@ -106,9 +104,7 @@ await evaluator.evaluate(text: string, grade: string) score: 'Slightly Complex' | 'Moderately Complex' | 'Very Complex' | 'Exceedingly Complex'; reasoning: string; metadata: { - promptVersion: string; model: string; - timestamp: Date; processingTimeMs: number; }; _internal: { @@ -195,9 +191,7 @@ await evaluator.evaluate(text: string) score: string; // e.g., 'K-1', '2-3', '4-5', '6-8', '9-10', '11-CCR' reasoning: string; metadata: { - promptVersion: string; model: string; - timestamp: Date; processingTimeMs: number; }; _internal: { diff --git a/sdks/typescript/src/evaluators/base.ts b/sdks/typescript/src/evaluators/base.ts index a910100..d4e48d6 100644 --- a/sdks/typescript/src/evaluators/base.ts +++ b/sdks/typescript/src/evaluators/base.ts @@ -158,7 +158,7 @@ export abstract class BaseEvaluator { // Initialize telemetry if enabled if (this.config.telemetry.enabled) { this.telemetryClient = new TelemetryClient({ - endpoint: 'https://api.learningcommons.org/v1/telemetry', + endpoint: 'https://api.learningcommons.org/evaluators-telemetry/v1/events', partnerKey: config.partnerKey, clientId: generateClientId(), enabled: true, diff --git a/sdks/typescript/src/evaluators/grade-level-appropriateness.ts b/sdks/typescript/src/evaluators/grade-level-appropriateness.ts index 3f43a61..9c5ad0a 100644 --- a/sdks/typescript/src/evaluators/grade-level-appropriateness.ts +++ b/sdks/typescript/src/evaluators/grade-level-appropriateness.ts @@ -2,10 +2,10 @@ import type { LLMProvider } from '../providers/index.js'; import { createProvider } from '../providers/index.js'; import { GradeLevelAppropriatenessSchema, - type GradeLevelAppropriateness, + type GradeLevelAppropriatenessInternal, } from '../schemas/grade-level-appropriateness.js'; import { getSystemPrompt, getUserPrompt } from '../prompts/grade-level-appropriateness/index.js'; -import type { EvaluationResult } from '../schemas/index.js'; +import type { EvaluationResult, GradeBand } from '../schemas/index.js'; import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; import { ValidationError, wrapProviderError } from '../errors.js'; @@ -69,7 +69,7 @@ export class GradeLevelAppropriatenessEvaluator extends BaseEvaluator { * @throws {ValidationError} If text is empty or too short/long * @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError) */ - async evaluate(text: string): Promise> { + async evaluate(text: string): Promise> { this.logger.info('Starting grade level appropriateness evaluation', { evaluator: 'grade-level-appropriateness', operation: 'evaluate', @@ -107,9 +107,7 @@ export class GradeLevelAppropriatenessEvaluator extends BaseEvaluator { score: response.data.grade, reasoning: response.data.reasoning, metadata: { - promptVersion: '1.2.0', model: 'google:gemini-2.5-pro', - timestamp: new Date(), processingTimeMs: latencyMs, }, _internal: response.data, @@ -186,7 +184,7 @@ export class GradeLevelAppropriatenessEvaluator extends BaseEvaluator { export async function evaluateGradeLevelAppropriateness( text: string, config: BaseEvaluatorConfig -): Promise> { +): Promise> { const evaluator = new GradeLevelAppropriatenessEvaluator(config); return evaluator.evaluate(text); } diff --git a/sdks/typescript/src/evaluators/index.ts b/sdks/typescript/src/evaluators/index.ts index 3a42ba2..7451765 100644 --- a/sdks/typescript/src/evaluators/index.ts +++ b/sdks/typescript/src/evaluators/index.ts @@ -23,6 +23,5 @@ export { export { TextComplexityEvaluator, evaluateTextComplexity, - type TextComplexityScore, - type TextComplexityInternal, + type TextComplexityResult, } from './text-complexity.js'; diff --git a/sdks/typescript/src/evaluators/sentence-structure.ts b/sdks/typescript/src/evaluators/sentence-structure.ts index c1b66d2..689ec87 100644 --- a/sdks/typescript/src/evaluators/sentence-structure.ts +++ b/sdks/typescript/src/evaluators/sentence-structure.ts @@ -6,6 +6,7 @@ import { type SentenceAnalysis, type SentenceFeatures, type ComplexityClassification, + type SentenceStructureInternal, } from '../schemas/sentence-structure.js'; import { calculateReadabilityMetrics, addEngineeredFeatures, featuresToJSON } from '../features/index.js'; import { @@ -14,39 +15,29 @@ import { getSystemPromptComplexity, getUserPromptComplexity, } from '../prompts/sentence-structure/index.js'; -import type { EvaluationResult, ComplexityLevel } from '../schemas/index.js'; +import type { EvaluationResult, TextComplexityLevel } from '../schemas/index.js'; import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; import type { StageDetail } from '../telemetry/index.js'; import { ValidationError, wrapProviderError } from '../errors.js'; -/** - * Internal data structure for sentence structure evaluation - */ -interface SentenceStructureInternal { - sentenceAnalysis: SentenceAnalysis; - features: SentenceFeatures; - complexity: ComplexityClassification; -} - /** * Normalize complexity label to handle LLM output variations - * Ported from Python normalize_label function */ -function normalizeLabel(label: string | null | undefined): string | null { +function normalizeLabel(label: string | null | undefined): TextComplexityLevel | null { if (!label) { return null; } - const normalized = label.trim().toLowerCase(); - const mapping: Record = { - 'slightly complex': 'Slightly Complex', - 'moderately complex': 'Moderately Complex', - 'very complex': 'Very Complex', - 'exceedingly complex': 'Exceedingly Complex', - 'extremely complex': 'Exceedingly Complex', // Maps to Exceedingly Complex + const normalized = label.trim().toLowerCase().replace(/_/g, ' '); + const mapping: Record = { + 'slightly complex': 'Slightly complex', + 'moderately complex': 'Moderately complex', + 'very complex': 'Very complex', + 'exceedingly complex': 'Exceedingly complex', + 'extremely complex': 'Exceedingly complex', }; - return mapping[normalized] || null; // Return null if no mapping found + return mapping[normalized] ?? null; } /** @@ -57,11 +48,11 @@ function normalizeLabel(label: string | null | undefined): string | null { * 1. Analyze grammatical structure (sentence types, clauses, phrases, etc.) * 2. Classify complexity using features and grade-specific rubric * - * Based on SCASS Text Complexity rubric with 4 levels: - * - Slightly Complex - * - Moderately Complex - * - Very Complex - * - Exceedingly Complex + * Based on Qualitative Text Complexity rubric with 4 levels: + * - Slightly complex + * - Moderately complex + * - Very complex + * - Exceedingly complex * * @example * ```typescript @@ -70,7 +61,7 @@ function normalizeLabel(label: string | null | undefined): string | null { * }); * * const result = await evaluator.evaluate(text, "3"); - * console.log(result.score); // "Moderately Complex" + * console.log(result.score); // "Moderately complex" * console.log(result.reasoning); * ``` */ @@ -119,7 +110,7 @@ export class SentenceStructureEvaluator extends BaseEvaluator { async evaluate( text: string, grade: string - ): Promise> { + ): Promise> { this.logger.info('Starting sentence structure evaluation', { evaluator: 'sentence-structure', operation: 'evaluate', @@ -183,9 +174,7 @@ export class SentenceStructureEvaluator extends BaseEvaluator { score: complexityResponse.data.answer, reasoning: complexityResponse.data.reasoning, metadata: { - promptVersion: '1.2.0', model: 'openai:gpt-4o', - timestamp: new Date(), processingTimeMs: latencyMs, }, _internal: { @@ -338,7 +327,7 @@ export class SentenceStructureEvaluator extends BaseEvaluator { return { data: { ...response.data, - answer: normalizedAnswer as ComplexityLevel, + answer: normalizedAnswer, }, usage: response.usage, latencyMs: response.latencyMs, @@ -364,7 +353,7 @@ export async function evaluateSentenceStructure( text: string, grade: string, config: BaseEvaluatorConfig -): Promise> { +): Promise> { const evaluator = new SentenceStructureEvaluator(config); return evaluator.evaluate(text, grade); } diff --git a/sdks/typescript/src/evaluators/text-complexity.ts b/sdks/typescript/src/evaluators/text-complexity.ts index 5493a72..114ba03 100644 --- a/sdks/typescript/src/evaluators/text-complexity.ts +++ b/sdks/typescript/src/evaluators/text-complexity.ts @@ -1,27 +1,19 @@ import pLimit from 'p-limit'; import { VocabularyEvaluator } from './vocabulary.js'; import { SentenceStructureEvaluator } from './sentence-structure.js'; +import type { SentenceStructureInternal } from '../schemas/sentence-structure.js'; import type { BaseEvaluatorConfig } from './base.js'; import { BaseEvaluator } from './base.js'; -import type { EvaluationResult } from '../schemas/index.js'; +import type { EvaluationResult, TextComplexityLevel } from '../schemas/index.js'; +import type { VocabularyInternal } from '../schemas/vocabulary.js'; /** - * Internal data structure for text complexity evaluation - * Stores either successful evaluation results or errors from sub-evaluators + * Result map returned by TextComplexityEvaluator. + * Each key holds the full evaluation result from its sub-evaluator, or an error if it failed. */ -export interface TextComplexityInternal { - vocabulary: EvaluationResult | { error: Error }; - sentenceStructure: EvaluationResult | { error: Error }; -} - -/** - * Composite score for text complexity - */ -export interface TextComplexityScore { - /** Vocabulary complexity score */ - vocabulary: string; - /** Sentence structure complexity score */ - sentenceStructure: string; +export interface TextComplexityResult { + vocabulary: EvaluationResult | { error: Error }; + sentenceStructure: EvaluationResult | { error: Error }; } /** @@ -42,8 +34,9 @@ export interface TextComplexityScore { * }); * * const result = await evaluator.evaluate(text, "5"); - * console.log(result.score.vocabulary); - * console.log(result.score.sentenceStructure); + * if (!('error' in result.vocabulary)) { + * console.log(result.vocabulary.score); // "Moderately complex" + * } * ``` */ export class TextComplexityEvaluator extends BaseEvaluator { @@ -76,16 +69,16 @@ export class TextComplexityEvaluator extends BaseEvaluator { * Evaluate text complexity for a given text and grade level * * Runs vocabulary and sentence structure evaluations in parallel with concurrency control. + * If both sub-evaluators fail, throws an error. Otherwise returns a result map where + * failed sub-evaluators are represented as `{ error: Error }`. * * @param text - The text to evaluate * @param grade - The target grade level (3-12) - * @returns Evaluation result with composite complexity score - * @throws {Error} If text is empty or grade is invalid + * @returns Map of sub-evaluator results + * @throws {ValidationError} If text is empty or grade is invalid + * @throws {Error} If all sub-evaluators fail */ - async evaluate( - text: string, - grade: string - ): Promise> { + async evaluate(text: string, grade: string): Promise { this.logger.info('Starting text complexity evaluation', { evaluator: 'text-complexity', operation: 'evaluate', @@ -100,29 +93,23 @@ export class TextComplexityEvaluator extends BaseEvaluator { const startTime = Date.now(); // Run both evaluators in parallel with concurrency control - const [vocabResult, sentenceResult] = await Promise.all([ + const [vocabResult, sentenceResult]: [ + EvaluationResult | { error: Error }, + EvaluationResult | { error: Error }, + ] = await Promise.all([ this.limit(() => this.runSubEvaluator(this.vocabularyEvaluator, text, grade)), this.limit(() => this.runSubEvaluator(this.sentenceStructureEvaluator, text, grade)), ]); const latencyMs = Date.now() - startTime; - - // Build combined reasoning - const reasoning = this.buildCombinedReasoning(vocabResult, sentenceResult); - - // Check if any evaluations failed const vocabFailed = 'error' in vocabResult; const sentenceFailed = 'error' in sentenceResult; const hasFailures = vocabFailed || sentenceFailed; if (hasFailures) { const errors: string[] = []; - if (vocabFailed) { - errors.push(`Vocabulary evaluation failed: ${vocabResult.error.message}`); - } - if (sentenceFailed) { - errors.push(`Sentence structure evaluation failed: ${sentenceResult.error.message}`); - } + if (vocabFailed) errors.push(`Vocabulary: ${vocabResult.error.message}`); + if (sentenceFailed) errors.push(`Sentence structure: ${sentenceResult.error.message}`); this.logger.error('Text complexity evaluation completed with errors', { evaluator: 'text-complexity', @@ -132,32 +119,11 @@ export class TextComplexityEvaluator extends BaseEvaluator { processingTimeMs: latencyMs, }); - // If both failed, throw error if (vocabFailed && sentenceFailed) { - throw new Error( - `Text complexity evaluation failed: ${errors.join('; ')}` - ); + throw new Error(`Text complexity evaluation failed: ${errors.join('; ')}`); } } - const result = { - score: { - vocabulary: vocabFailed ? 'N/A' : vocabResult.score, - sentenceStructure: sentenceFailed ? 'N/A' : sentenceResult.score, - }, - reasoning, - metadata: { - promptVersion: '1.0', - model: 'composite:gemini-2.5-pro+gpt-4o', - timestamp: new Date(), - processingTimeMs: latencyMs, - }, - _internal: { - vocabulary: vocabResult, - sentenceStructure: sentenceResult, - }, - }; - // Send telemetry (fire-and-forget) this.sendTelemetry({ status: hasFailures ? 'error' : 'success', @@ -179,49 +145,23 @@ export class TextComplexityEvaluator extends BaseEvaluator { hasFailures, }); - return result; + return { vocabulary: vocabResult, sentenceStructure: sentenceResult }; } /** - * Run a sub-evaluator with error handling - * Returns the evaluation result or an error object + * Run a sub-evaluator with error handling. + * Returns the evaluation result or `{ error: Error }` if the evaluator throws. */ - private async runSubEvaluator( - evaluator: { evaluate(text: string, grade: string): Promise> }, + private async runSubEvaluator( + evaluator: { evaluate(text: string, grade: string): Promise> }, text: string, grade: string - ): Promise | { error: Error }> { + ): Promise | { error: Error }> { try { return await evaluator.evaluate(text, grade); } catch (error) { - return { - error: error instanceof Error ? error : new Error(String(error)), - }; - } - } - - /** - * Build combined reasoning from individual results - */ - private buildCombinedReasoning( - vocabResult: EvaluationResult | { error: Error }, - sentenceResult: EvaluationResult | { error: Error } - ): string { - const parts: string[] = []; - - if ('error' in vocabResult) { - parts.push(`Vocabulary Complexity: Evaluation failed - ${vocabResult.error.message}`); - } else { - parts.push(`Vocabulary Complexity (${vocabResult.score}):\n${vocabResult.reasoning}`); + return { error: error instanceof Error ? error : new Error(String(error)) }; } - - if ('error' in sentenceResult) { - parts.push(`Sentence Structure Complexity: Evaluation failed - ${sentenceResult.error.message}`); - } else { - parts.push(`Sentence Structure Complexity (${sentenceResult.score}):\n${sentenceResult.reasoning}`); - } - - return parts.join('\n\n'); } } @@ -244,7 +184,7 @@ export async function evaluateTextComplexity( text: string, grade: string, config: BaseEvaluatorConfig -): Promise> { +): Promise { const evaluator = new TextComplexityEvaluator(config); return evaluator.evaluate(text, grade); } diff --git a/sdks/typescript/src/evaluators/vocabulary.ts b/sdks/typescript/src/evaluators/vocabulary.ts index 9b145fa..f1733a7 100644 --- a/sdks/typescript/src/evaluators/vocabulary.ts +++ b/sdks/typescript/src/evaluators/vocabulary.ts @@ -2,7 +2,7 @@ import type { LLMProvider } from '../providers/index.js'; import { createProvider } from '../providers/index.js'; import { VocabularyComplexitySchema, - type VocabularyComplexity, + type VocabularyInternal, type BackgroundKnowledge, } from '../schemas/vocabulary.js'; import { calculateFleschKincaidGrade } from '../features/index.js'; @@ -11,7 +11,7 @@ import { getSystemPrompt, getUserPrompt, } from '../prompts/vocabulary/index.js'; -import type { EvaluationResult } from '../schemas/index.js'; +import type { EvaluationResult, TextComplexityLevel } from '../schemas/index.js'; import { BaseEvaluator, type BaseEvaluatorConfig } from './base.js'; import type { StageDetail } from '../telemetry/index.js'; import { ValidationError, wrapProviderError } from '../errors.js'; @@ -38,7 +38,7 @@ import { ValidationError, wrapProviderError } from '../errors.js'; * }); * * const result = await evaluator.evaluate(text, "3"); - * console.log(result.score); // "moderately complex" + * console.log(result.score); // "Moderately complex" * console.log(result.reasoning); * ``` */ @@ -97,7 +97,7 @@ export class VocabularyEvaluator extends BaseEvaluator { async evaluate( text: string, grade: string - ): Promise> { + ): Promise> { this.logger.info('Starting vocabulary evaluation', { evaluator: 'vocabulary', operation: 'evaluate', @@ -166,9 +166,7 @@ export class VocabularyEvaluator extends BaseEvaluator { score: complexityResponse.data.complexity_score, reasoning: complexityResponse.data.reasoning, metadata: { - promptVersion: '1.2.0', model: `openai:gpt-4o-2024-11-20 + ${complexityProviderName}`, - timestamp: new Date(), processingTimeMs: latencyMs, }, _internal: complexityResponse.data, @@ -281,7 +279,7 @@ export class VocabularyEvaluator extends BaseEvaluator { grade: string, backgroundKnowledge: string, fkLevel: number - ): Promise<{ data: VocabularyComplexity; usage: { inputTokens: number; outputTokens: number }; latencyMs: number }> { + ): Promise<{ data: VocabularyInternal; usage: { inputTokens: number; outputTokens: number }; latencyMs: number }> { const systemPrompt = getSystemPrompt(grade); const userPrompt = getUserPrompt(text, grade, backgroundKnowledge, fkLevel); @@ -326,7 +324,7 @@ export async function evaluateVocabulary( text: string, grade: string, config: BaseEvaluatorConfig -): Promise> { +): Promise> { const evaluator = new VocabularyEvaluator(config); return evaluator.evaluate(text, grade); } diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index 93da1d0..51f764c 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -2,12 +2,10 @@ export type { EvaluationResult, EvaluationMetadata, - BatchEvaluationResult, - BatchSummary, EvaluationError, } from './schemas/index.js'; -export { ComplexityLevel, GradeLevel, GradeBand } from './schemas/index.js'; +export { TextComplexityLevel, GradeBand } from './schemas/index.js'; // Error types export { @@ -40,6 +38,7 @@ export type { SentenceAnalysis, ComplexityClassification, SentenceFeatures, + SentenceStructureInternal, } from './schemas/sentence-structure.js'; export { @@ -48,17 +47,13 @@ export { } from './schemas/sentence-structure.js'; // Vocabulary exports -export type { - VocabularyComplexity, - VocabularyComplexityLevel, -} from './schemas/vocabulary.js'; +export type { VocabularyInternal } from './schemas/vocabulary.js'; // Grade Level Appropriateness exports -export type { GradeLevelAppropriateness } from './schemas/grade-level-appropriateness.js'; +export type { GradeLevelAppropriatenessInternal } from './schemas/grade-level-appropriateness.js'; export { GradeLevelAppropriatenessSchema } from './schemas/grade-level-appropriateness.js'; - export { VocabularyEvaluator, evaluateVocabulary, @@ -68,8 +63,7 @@ export { evaluateGradeLevelAppropriateness, TextComplexityEvaluator, evaluateTextComplexity, - type TextComplexityScore, - type TextComplexityInternal, + type TextComplexityResult, type BaseEvaluatorConfig, type TelemetryOptions, type EvaluatorMetadata, diff --git a/sdks/typescript/src/schemas/grade-level-appropriateness.ts b/sdks/typescript/src/schemas/grade-level-appropriateness.ts index e23e638..31af558 100644 --- a/sdks/typescript/src/schemas/grade-level-appropriateness.ts +++ b/sdks/typescript/src/schemas/grade-level-appropriateness.ts @@ -24,4 +24,4 @@ export const GradeLevelAppropriatenessSchema = z.object({ .describe('Scaffolding needed for the text to be appropriate for the alternative grade'), }); -export type GradeLevelAppropriateness = z.infer; +export type GradeLevelAppropriatenessInternal = z.infer; diff --git a/sdks/typescript/src/schemas/index.ts b/sdks/typescript/src/schemas/index.ts index f1b73d3..22784fe 100644 --- a/sdks/typescript/src/schemas/index.ts +++ b/sdks/typescript/src/schemas/index.ts @@ -1,15 +1,12 @@ export { - ComplexityLevel, - GradeLevel, + TextComplexityLevel, type EvaluationResult, type EvaluationMetadata, - type BatchEvaluationResult, - type BatchSummary, type EvaluationError, } from './outputs.js'; export { GradeBand, GradeLevelAppropriatenessSchema, - type GradeLevelAppropriateness, + type GradeLevelAppropriatenessInternal, } from './grade-level-appropriateness.js'; diff --git a/sdks/typescript/src/schemas/outputs.ts b/sdks/typescript/src/schemas/outputs.ts index 9ab807e..b0e4770 100644 --- a/sdks/typescript/src/schemas/outputs.ts +++ b/sdks/typescript/src/schemas/outputs.ts @@ -1,36 +1,23 @@ import { z } from 'zod'; /** - * Complexity levels for sentence structure evaluation + * Shared complexity levels used across all text complexity evaluators + * (Vocabulary, Sentence Structure, and any future sub-evaluators) */ -export const ComplexityLevel = z.enum([ - 'Slightly Complex', - 'Moderately Complex', - 'Very Complex', - 'Exceedingly Complex', +export const TextComplexityLevel = z.enum([ + 'Slightly complex', + 'Moderately complex', + 'Very complex', + 'Exceedingly complex', ]); -export type ComplexityLevel = z.infer; - -/** - * Grade levels for vocabulary evaluation - */ -export const GradeLevel = z.enum([ - 'Below Grade Level', - 'At Grade Level', - 'Above Grade Level', -]); - -export type GradeLevel = z.infer; +export type TextComplexityLevel = z.infer; /** * Metadata attached to all evaluation results */ export interface EvaluationMetadata { - evaluatorVersion?: string; - promptVersion: string; model: string; - timestamp: Date; processingTimeMs: number; } @@ -63,7 +50,6 @@ export interface EvaluationError { text: string; grade?: string; }; - timestamp: Date; } /** diff --git a/sdks/typescript/src/schemas/sentence-structure.ts b/sdks/typescript/src/schemas/sentence-structure.ts index 4f9522f..b912b68 100644 --- a/sdks/typescript/src/schemas/sentence-structure.ts +++ b/sdks/typescript/src/schemas/sentence-structure.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { ComplexityLevel } from './outputs.js'; +import { TextComplexityLevel } from './outputs.js'; /** * Stage 1: Detailed sentence analysis output (40+ metrics) @@ -69,11 +69,20 @@ export type SentenceAnalysis = z.infer; */ export const ComplexityClassificationSchema = z.object({ reasoning: z.string().describe('Detailed pedagogically appropriate reasoning'), - answer: ComplexityLevel, + answer: TextComplexityLevel, }); export type ComplexityClassification = z.infer; +/** + * Internal data structure for sentence structure evaluation + */ +export interface SentenceStructureInternal { + sentenceAnalysis: SentenceAnalysis; + features: SentenceFeatures; + complexity: ComplexityClassification; +} + /** * Engineered features computed from sentence analysis * These are calculated in TypeScript, not requested from LLM diff --git a/sdks/typescript/src/schemas/vocabulary.ts b/sdks/typescript/src/schemas/vocabulary.ts index f5b80d0..0badcad 100644 --- a/sdks/typescript/src/schemas/vocabulary.ts +++ b/sdks/typescript/src/schemas/vocabulary.ts @@ -1,33 +1,21 @@ import { z } from 'zod'; +import { TextComplexityLevel } from './outputs.js'; /** - * Vocabulary complexity levels matching Qual Text Complexity rubric (SAP) - */ -export const VocabularyComplexityLevel = z.enum([ - 'slightly complex', - 'moderately complex', - 'very complex', - 'exceedingly complex', -]); - -export type VocabularyComplexityLevel = z.infer; - -/** - * Vocabulary complexity evaluation output - * Ported from Python Output BaseModel + * Vocabulary evaluation output schema */ export const VocabularyComplexitySchema = z.object({ tier_2_words: z.string().describe('List of Tier 2 words (academic words)'), tier_3_words: z.string().describe('List of Tier 3 words (domain-specific)'), archaic_words: z.string().describe('List of Archaic words'), other_complex_words: z.string().describe('List of Other Complex words'), - complexity_score: VocabularyComplexityLevel.describe( + complexity_score: TextComplexityLevel.describe( 'The complexity of the text vocabulary' ), reasoning: z.string().describe('Detailed reasoning for the complexity rating'), }); -export type VocabularyComplexity = z.infer; +export type VocabularyInternal = z.infer; /** * Background knowledge assumption for a student at a given grade level diff --git a/sdks/typescript/tests/integration/sentence-structure.integration.test.ts b/sdks/typescript/tests/integration/sentence-structure.integration.test.ts index a9c9bd5..a1ccb8b 100644 --- a/sdks/typescript/tests/integration/sentence-structure.integration.test.ts +++ b/sdks/typescript/tests/integration/sentence-structure.integration.test.ts @@ -33,39 +33,39 @@ const TEST_CASES: BaseTestCase[] = [ // id: 'SS2', // grade: '2', // text: "The Roman Empire was a powerful empire that lasted for hundreds of years. It started as a small village in Italy and grew into a huge empire that controlled much of Europe, Asia, and Africa. The Roman Empire had many strong leaders like Julius Caesar and Augustus. These leaders helped the empire grow and become very powerful.\n \n\n The Roman Empire had a period of peace and prosperity called the Pax Romana. This time was good for the empire, but it didn't last forever. The empire started to have problems. The army became weaker, and the economy had problems. The empire was also attacked by groups of people called barbarians.\n \n\n The Roman Empire was divided into two parts: the Western Roman Empire and the Eastern Roman Empire. The Western Roman Empire eventually fell apart in 476 AD. The Eastern Roman Empire, also known as the Byzantine Empire, lasted for many more years. The Roman Empire left behind many things that we still use today, like the Roman alphabet and the calendar.", - // expected: 'moderately complex', - // acceptable: ['slightly complex', 'very complex'], + // expected: 'Moderately complex', + // acceptable: ['Slightly complex', 'Very complex'], // }, { id: 'SS3', grade: '3', text: "The hoisting gear consists of a double system of chains 13/16 in. in diameter placed side by side; each chain is anchored by an adjustable screw to the end of the jib, and, passing round the traveling carriage and down to the falling block, is taken along the jib over a sliding pulley which leads it on to the grooved barrel, 3 ft. 9 in. in diameter. In front of the barrel is placed an automatic winder which insures a proper coiling of the chain in the grooves. The motive power is derived from two cylinders 10 in. in diameter and 16 in. stroke, one being bolted to each side frame; these cylinders, which are provided with link motion and reversing gear, drive a steel crank shaft 2¾ in. in diameter; on this shaft is a steel sliding pinion which drives the barrel by a double purchase.", - expected: 'exceedingly complex', - acceptable: ['very complex'], + expected: 'Exceedingly complex', + acceptable: ['Very complex'], }, { id: 'SS4', grade: '4', text: "Before corals bleach, they do not show many other signs of feeling stressed. So, if we want to understand a coral's health, we have to study its cells. Inside cells we have a lot of information, including DNA, RNA, and proteins. These molecules can help us find clues about the communication between the coral and the algae. But also, these molecules can teach us how to know when corals are stressed.\nWhen an organism is stressed, every cell in its body will react. Everything will do its best to survive! In response to stress, the cell will use its DNA to make RNA, so that it can then make proteins that will fight off the stress. If an organism has been stressed before, it can respond to the stress faster and better. Think of it like visiting a city: the first time you visit, you will need a map to find your hotel. The more often you visit the city, the less you will need the map because you will remember, and you will get back to the hotel faster.", - expected: 'very complex', - acceptable: ['moderately complex', 'exceedingly complex'], + expected: 'Very complex', + acceptable: ['Moderately complex', 'Exceedingly complex'], }, { id: 'SS5', grade: '5', text: "Mesopotamia, located in present-day Iraq, is known as the 'Cradle of Civilization' because it was home to some of the earliest civilizations in the world. The region got its name from the ancient Greek words for 'land between the rivers,' referring to the Tigris and Euphrates rivers. These rivers provided water for the fertile land, making it perfect for farming. The regular flooding of the rivers made the land around them ideal for growing crops, which helped people settle down and form permanent villages. These villages eventually grew into cities, where people developed many of the characteristics of civilization, like organized government, complex buildings, and different social classes.\n \n\n The first civilizations in Mesopotamia were the Sumerians, who lived around 5,000 years ago. They invented the world's first written language, called cuneiform, which they used to keep track of things like food supplies and trade. They also developed a system of numbers, which helped them with math and measurement. The Sumerians built impressive cities like Ur, Eridu, and Uruk, which had populations of over 50,000 people. These cities were centers of learning and culture, and they helped spread knowledge and ideas throughout the region.\n \n\n Over time, other civilizations rose and fell in Mesopotamia, including the Akkadians, Babylonians, and Assyrians. Each civilization made its own contributions to the development of human society. The Babylonians are famous for their code of laws, which was one of the first written legal systems in the world. The Assyrians were known for their powerful military and their impressive palaces. Mesopotamia's history is full of amazing inventions and innovations that shaped the world we live in today.\n \n\n The development of civilization in Mesopotamia was not just about the fertile land and the rivers. Changes in climate and the environment also played a role. People had to become more organized and work together to survive. This led to the development of complex societies and governments. Mesopotamia's story is a reminder of how human ingenuity and adaptability can lead to amazing achievements.\n \n\n The 'Cradle of Civilization' is a term that refers to the regions where the earliest known human civilizations emerged. Mesopotamia is a prime example of this, as it was a place where people learned to live together, build cities, and develop new technologies that changed the course of human history. The innovations that came from Mesopotamia, like writing, mathematics, and agriculture, continue to influence our lives today. By studying ancient Mesopotamia, we can learn about the origins of our own civilization and the challenges and triumphs of early humans.", - expected: 'slightly complex', - acceptable: ['moderately complex'], + expected: 'Slightly complex', + acceptable: ['Moderately complex'], // TODO: Valiadate the test-case with additional data from Grade 5 - // expected: 'exceedingly complex', - // acceptable: ['very complex'], + // expected: 'Exceedingly complex', + // acceptable: ['Very complex'], }, { id: 'SS6', grade: '6', text: "Benjamin Franklin was a very important person in American history. He was born in Boston, Massachusetts in 1706. He was one of 17 children. Franklin did not go to school for very long. He learned to be a printer from his brother. Franklin was a very smart man. He invented many things, like bifocals, the Franklin stove, and the lightning rod. He also started the first public library in Philadelphia. Franklin was a writer, too. He wrote a book called *Poor Richard's Almanack*. It had many famous sayings, like \"Lost Time is never found again.\"\n\nFranklin was also a politician. He helped write the Declaration of Independence. He was a diplomat, too. He helped the United States get help from France during the Revolutionary War. He was a very busy man! Franklin was a scientist, a writer, a politician, and an inventor. He was a very important person in American history.\n\nFranklin was a very interesting person. He was a scientist who did experiments with electricity. He was a writer who wrote a book of sayings. He was a politician who helped the United States become independent. He was a diplomat who helped the United States get help from other countries. He was a very busy man!\n\nFranklin was a very smart man. He was a self-taught man who learned a lot on his own. He was a very creative man who invented many things. He was a very kind man who helped others. He was a very important man who helped shape the United States.\n\nFranklin was a very influential person. He was a leader who helped people. He was a thinker who came up with new ideas. He was a writer who shared his thoughts with others. He was a scientist who helped people understand the world. He was a very important person who helped make the United States what it is today.", - expected: 'slightly complex', - acceptable: ['moderately complex'], + expected: 'Slightly complex', + acceptable: ['Moderately complex'], }, ]; diff --git a/sdks/typescript/tests/integration/vocabulary.integration.test.ts b/sdks/typescript/tests/integration/vocabulary.integration.test.ts index 49b4662..4a60cac 100644 --- a/sdks/typescript/tests/integration/vocabulary.integration.test.ts +++ b/sdks/typescript/tests/integration/vocabulary.integration.test.ts @@ -34,50 +34,50 @@ const TEST_CASES: BaseTestCase[] = [ id: 'V3', grade: '3', text: 'Civil rights are rights that all people in a country have. The civil rights of a country apply to all the citizens within its borders. These rights are given by the laws of the country. Civil rights are sometimes thought to be the same as natural rights. In many countries civil rights include freedom of speech, freedom of the press, freedom of religion, and freedom of assembly. Civil rights also include the right to own property and the right to get fair and equal treatment from the government, from other citizens, and from private groups.', - expected: 'very complex', - acceptable: ['moderately complex', 'exceedingly complex'], + expected: 'Very complex', + acceptable: ['Moderately complex', 'Exceedingly complex'], }, { id: 'V4', grade: '4', text: 'Bluetooth is a protocol for wireless communication over short distances. It was developed in the 1990s, to reduce the number of cables. Devices such as mobile phones, laptops, PCs, printers, digital cameras and video game consoles can connect to each other, and exchange information. This is done using radio waves. It can be done securely. Bluetooth is only used for relatively short distances, like a few metres. There are different standards. Data rates vary. Currently, they are at 1-3 MBit per second.', - expected: 'exceedingly complex', - acceptable: ['very complex'], + expected: 'Exceedingly complex', + acceptable: ['Very complex'], }, { id: 'V5', grade: '5', text: `The scientific method is a way to learn about the world around us. It helps us figure out how things work. Scientists use the scientific method to test their ideas. They start by making observations and asking questions. Then, they make a guess, or a hypothesis, about what might be the answer. They use their hypothesis to make predictions about what will happen in an experiment. Scientists then test their predictions by doing experiments. If the results of the experiment match their predictions, then their hypothesis is supported. If the results don't match, then they need to change their hypothesis. Scientists repeat this process many times to make sure their hypothesis is correct. The scientific method is important because it helps us learn new things. It helps us understand the world around us. Scientists use the scientific method to make new discoveries and solve problems.`, - expected: 'slightly complex', - acceptable: ['moderately complex'], + expected: 'Slightly complex', + acceptable: ['Moderately complex'], }, { id: 'V6', grade: '6', text: `Chicago in 1871 was a city ready to burn. The city boasted having 59,500 buildings, many of them—such as the Courthouse and the Tribune Building—large and ornately decorated. The trouble was that about two-thirds of all these structures were made entirely of wood. Many of the remaining buildings (even the ones proclaimed to be 'fireproof') looked solid, but were actually jerrybuilt affairs; the stone or brick exteriors hid wooden frames and floors, all topped with highly flammable tar or shingle roofs. It was also a common practice to disguise wood as another kind of building material. The fancy exterior decorations on just about every building were carved from wood, then painted to look like stone or marble.`, - expected: 'very complex', - acceptable: ['moderately complex', 'exceedingly complex'], + expected: 'Very complex', + acceptable: ['Moderately complex', 'Exceedingly complex'], }, { id: 'V7', grade: '7', text: `The scientific method is a way of learning about the world around us. It's a process that helps us understand how things work and why they happen. It's not just for scientists; we all use the scientific method in our everyday lives, even if we don't realize it. The scientific method starts with an observation. We notice something interesting and want to know more about it. For example, you might notice that your plant is wilting. You might wonder why this is happening. Next, we form a hypothesis, which is a possible explanation for our observation. In our plant example, you might hypothesize that the plant is wilting because it needs more water. Then, we test our hypothesis by doing an experiment. We change something in our experiment to see if it affects the outcome. In our plant example, you could water the plant and see if it recovers. Based on the results of our experiment, we can either support or reject our hypothesis. If the plant recovers after being watered, then your hypothesis is supported. If the plant doesn't recover, then you need to come up with a new hypothesis. The scientific method is a powerful tool for learning and understanding the world around us. It's a process of asking questions, testing ideas, and drawing conclusions based on evidence. It's a way of thinking that helps us to be curious, to be critical, and to be open to new ideas.`, - expected: 'slightly complex', - acceptable: ['moderately complex'], + expected: 'Slightly complex', + acceptable: ['Moderately complex'], }, { id: 'V8', grade: '8', text: 'The American Revolution was a war for independence between the thirteen American colonies and Great Britain. The war started in 1775 and ended in 1783. The colonists wanted to be free from British rule. They wanted to make their own laws and govern themselves. The colonists were angry about new taxes that the British Parliament imposed on them. They felt that they were being taxed without having a say in how the money was spent. The colonists also felt that the British government was not treating them fairly. The war began with the Battles of Lexington and Concord in April 1775. The colonists, led by General George Washington, fought against the British army. The war was long and difficult, but the colonists eventually won. The colonists won the war because they had the support of the French. The French helped the colonists by providing them with soldiers, ships, and money. The colonists also had a strong leader in George Washington. He was a skilled military leader and he inspired the colonists to fight for their freedom. The American Revolution was a turning point in history. It showed that colonies could break free from their mother countries and become independent nations. The American Revolution also inspired other revolutions around the world.', - expected: 'slightly complex', - acceptable: ['moderately complex'], + expected: 'Slightly complex', + acceptable: ['Moderately complex'], }, { id: 'V9', grade: '9', text: `Mr. President: I would like to speak briefly and simply about a serious national condition. It is a national feeling of fear and frustration that could result in national suicide and the end of everything that we Americans hold dear. It is a condition that comes from the lack of effective leadership in either the Legislative Branch or the Executive Branch of our Government. That leadership is so lacking that serious and responsible proposals are being made that national advisory commissions be appointed to provide such critically needed leadership. I speak as briefly as possible because too much harm has already been done with irresponsible words of bitterness and selfish political opportunism. I speak as briefly as possible because the issue is too great to be obscured by eloquence. I speak simply and briefly in the hope that my words will be taken to heart. I speak as a Republican. I speak as a woman. I speak as a United States Senator. I speak as an American. The United States Senate has long enjoyed worldwide respect as the greatest deliberative body in the world. But recently that deliberative character has too often been debased to the level of a forum of hate and character assassination sheltered by the shield of congressional immunity. It is ironical that we Senators can in debate in the Senate directly or indirectly, by any form of words, impute to any American who is not a Senator any conduct or motive unworthy or unbecoming an American—and without that non-Senator American having any legal redress against us—yet if we say the same thing in the Senate about our colleagues we can be stopped on the grounds of being out of order. It is strange that we can verbally attack anyone else without restraint and with full protection and yet we hold ourselves above the same type of criticism here on the Senate Floor. Surely the United States Senate is big enough to take self-criticism and self-appraisal. Surely we should be able to take the same kind of character attacks that we "dish out" to outsiders. I think that it is high time for the United States Senate and its members to do some soul-searching—for us to weigh our consciences—on the manner in which we are performing our duty to the people of America—on the manner in which we are using or abusing our individual powers and privileges. I think that it is high time that we remembered that we have sworn to uphold and defend the Constitution. I think that it is high time that we remembered that the Constitution, as amended, speaks not only of the freedom of speech but also of trial by jury instead of trial by accusation. Whether it be a criminal prosecution in court or a character prosecution in the Senate, there is little practical distinction when the life of a person has been ruined. Those of us who shout the loudest about Americanism in making character assassinations are all too frequently those who, by our own words and acts, ignore some of the basic principles of Americanism: The right to criticize; The right to hold unpopular beliefs; The right to protest; The right of independent thought. The exercise of these rights should not cost one single American citizen his reputation or his right to a livelihood nor should he be in danger of losing his reputation or livelihood merely because he happens to know someone who holds unpopular beliefs. Who of us doesn't? Otherwise none of us could call our souls our own. Otherwise thought control would have set in. The American people are sick and tired of being afraid to speak their minds lest they be politically smeared as "Communists" or "Fascists" by their opponents. Freedom of speech is not what it used to be in America. It has been so abused by some that it is not exercised by others. The American people are sick and tired of seeing innocent people smeared and guilty people whitewashed. But there have been enough proved cases, such as the Amerasia case, the Hiss case, the Coplon case, the Gold case, to cause the nationwide distrust and strong suspicion that there may be something to the unproved, sensational accusations. I doubt if the Republican Party could—simply because I don't believe the American people will uphold any political party that puts political exploitation above national interest. Surely we Republicans aren't that desperate for victory. I don't want to see the Republican Party win that way. While it might be a fleeting victory for the Republican Party, it would be a more lasting defeat for the American people. Surely it would ultimately be suicide for the Republican Party and the two-party system that has protected our American liberties from the dictatorship of a one-party system. As members of the Minority Party, we do not have the primary authority to formulate the policy of our Government. But we do have the responsibility of rendering constructive criticism, of clarifying issues, of allaying fears by acting as responsible citizens. As a woman, I wonder how the mothers, wives, sisters, and daughters feel about the way in which members of their families have been politically mangled in the Senate debate—and I use the word "debate" advisedly. As a United States Senator, I am not proud of the way in which the Senate has been made a publicity platform for irresponsible sensationalism. I am not proud of the reckless abandon in which unproved charges have been hurled from the side of the aisle. I am not proud of the obviously staged, undignified countercharges that have been attempted in retaliation from the other side of the aisle. I don't like the way the Senate has been made a rendezvous for vilification, for selfish political gain at the sacrifice of individual reputations and national unity. I am not proud of the way we smear outsiders from the Floor of the Senate and hide behind the cloak of congressional immunity and still place ourselves beyond criticism on the Floor of the Senate. As an American, I am shocked at the way Republicans and Democrats alike are playing directly into the Communist design of "confuse, divide, and conquer." As an American, I don't want a Democratic Administration "whitewash" or "cover-up" any more than I want a Republican smear or witch hunt. As an American, I condemn a Republican "Fascist" just as much I condemn a Democratic "Communist." I condemn a Democrat "Fascist" just as much as I condemn a Republican "Communist." They are equally dangerous to you and me and to our country. As an American, I want to see our nation recapture the strength and unity it once had when we fought the enemy instead of ourselves. It is with these thoughts that I have drafted what I call a "Declaration of Conscience." I am gratified that Senator Tobey, Senator Aiken, Senator Morse, Senator Ives, Senator Thye, and Senator Hendrickson have concurred in that declaration and have authorized me to announce their concurrence.`, - expected: 'very complex', - acceptable: ['moderately complex', 'exceedingly complex'], + expected: 'Very complex', + acceptable: ['Moderately complex', 'Exceedingly complex'], }, ]; diff --git a/sdks/typescript/tests/unit/evaluators/grade-level-appropriateness.test.ts b/sdks/typescript/tests/unit/evaluators/grade-level-appropriateness.test.ts index 7e04d9b..915b9d7 100644 --- a/sdks/typescript/tests/unit/evaluators/grade-level-appropriateness.test.ts +++ b/sdks/typescript/tests/unit/evaluators/grade-level-appropriateness.test.ts @@ -165,15 +165,11 @@ describe('GradeLevelAppropriatenessEvaluator - Evaluation Flow', () => { expect(result._internal).toHaveProperty('reasoning'); // Verify metadata structure - expect(result.metadata).toHaveProperty('promptVersion'); expect(result.metadata).toHaveProperty('model'); - expect(result.metadata).toHaveProperty('timestamp'); expect(result.metadata).toHaveProperty('processingTimeMs'); // Verify metadata values - expect(result.metadata.promptVersion).toBe('1.2.0'); expect(result.metadata.model).toBe('google:gemini-2.5-pro'); - expect(result.metadata.timestamp).toBeInstanceOf(Date); expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); // Verify _internal values diff --git a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts index a4971e2..b3f99f3 100644 --- a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts +++ b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts @@ -128,7 +128,7 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { // Mock complexity classification response vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ data: { - answer: 'Slightly Complex', + answer: 'Slightly complex', reasoning: 'The text uses simple sentence structures appropriate for third grade.', }, model: 'gpt-4o', @@ -143,7 +143,7 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { const result = await evaluator.evaluate(testText, testGrade); // Verify result structure - expect(result.score).toBe('Slightly Complex'); + expect(result.score).toBe('Slightly complex'); expect(result.reasoning).toContain('simple sentence structures'); expect(result.metadata).toBeDefined(); expect(result.metadata.model).toBe('openai:gpt-4o'); @@ -217,7 +217,7 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ data: { - answer: 'Moderately Complex', + answer: 'Moderately complex', reasoning: 'Detailed reasoning here', }, model: 'gpt-4o', @@ -234,15 +234,11 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { expect(result).toHaveProperty('_internal'); // Verify metadata structure - expect(result.metadata).toHaveProperty('promptVersion'); expect(result.metadata).toHaveProperty('model'); - expect(result.metadata).toHaveProperty('timestamp'); expect(result.metadata).toHaveProperty('processingTimeMs'); // Verify metadata values - expect(result.metadata.promptVersion).toBe('1.2.0'); expect(result.metadata.model).toBe('openai:gpt-4o'); - expect(result.metadata.timestamp).toBeInstanceOf(Date); expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); }); }); diff --git a/sdks/typescript/tests/unit/evaluators/text-complexity.test.ts b/sdks/typescript/tests/unit/evaluators/text-complexity.test.ts index a3c48bf..1044611 100644 --- a/sdks/typescript/tests/unit/evaluators/text-complexity.test.ts +++ b/sdks/typescript/tests/unit/evaluators/text-complexity.test.ts @@ -16,7 +16,7 @@ vi.mock('../../../src/providers/index.js', () => ({ data: { complexity_score: 'moderately complex', reasoning: 'Test reasoning', - answer: 'Moderately Complex', + answer: 'Moderately complex', }, usage: { inputTokens: 100, outputTokens: 50 }, latencyMs: 100, @@ -106,24 +106,20 @@ describe('TextComplexityEvaluator', () => { // Mock the child evaluators' evaluate methods vocabSpy = vi.spyOn((evaluator as any).vocabularyEvaluator, 'evaluate').mockResolvedValue({ - score: 'moderately complex', + score: 'Moderately complex', reasoning: 'Vocabulary test reasoning', metadata: { - promptVersion: '1.0', model: 'gemini-2.5-pro + gpt-4o', - timestamp: new Date(), processingTimeMs: 100, }, _internal: {}, }); sentenceSpy = vi.spyOn((evaluator as any).sentenceStructureEvaluator, 'evaluate').mockResolvedValue({ - score: 'Moderately Complex', + score: 'Moderately complex', reasoning: 'Sentence structure test reasoning', metadata: { - promptVersion: '1.0', model: 'gpt-4o', - timestamp: new Date(), processingTimeMs: 100, }, _internal: {}, @@ -141,15 +137,10 @@ describe('TextComplexityEvaluator', () => { const result = await evaluator.evaluate(text, grade); expect(result).toBeDefined(); - expect(result.score).toBeDefined(); - expect(result.score.vocabulary).toBeDefined(); - expect(result.score.sentenceStructure).toBeDefined(); - expect(result.reasoning).toBeDefined(); - expect(result.metadata).toBeDefined(); - expect(result.metadata.model).toBe('composite:gemini-2.5-pro+gpt-4o'); - expect(result._internal).toBeDefined(); - expect(result._internal!.vocabulary).toBeDefined(); - expect(result._internal!.sentenceStructure).toBeDefined(); + expect(result.vocabulary).toBeDefined(); + expect(result.sentenceStructure).toBeDefined(); + expect('error' in result.vocabulary).toBe(false); + expect('error' in result.sentenceStructure).toBe(false); }); it('should validate text input', async () => { @@ -201,8 +192,8 @@ describe('TextComplexityEvaluator', () => { // Allow some overhead but should be significantly less than 200ms expect(duration).toBeLessThan(200); - expect('error' in result._internal!.vocabulary).toBe(false); - expect('error' in result._internal!.sentenceStructure).toBe(false); + expect('error' in result.vocabulary).toBe(false); + expect('error' in result.sentenceStructure).toBe(false); }); it('should handle partial failures gracefully', async () => { @@ -215,11 +206,9 @@ describe('TextComplexityEvaluator', () => { const result = await evaluator.evaluate(text, grade); expect(result).toBeDefined(); - expect('error' in result._internal!.vocabulary).toBe(true); - expect((result._internal!.vocabulary as { error: Error }).error).toBeDefined(); - expect('error' in result._internal!.sentenceStructure).toBe(false); - expect(result.score.vocabulary).toBe('N/A'); - expect(result.score.sentenceStructure).not.toBe('N/A'); + expect('error' in result.vocabulary).toBe(true); + expect((result.vocabulary as { error: Error }).error).toBeDefined(); + expect('error' in result.sentenceStructure).toBe(false); }); it('should throw when both evaluators fail', async () => { @@ -239,27 +228,23 @@ describe('TextComplexityEvaluator', () => { const text = 'The cat sat on the mat.'; const grade = '5'; - // Override vocabulary to return "moderately complex" + // Override vocabulary to return "Moderately complex" vocabSpy.mockResolvedValue({ - score: 'moderately complex', + score: 'Moderately complex', reasoning: 'Vocab reasoning', metadata: { - promptVersion: '1.0', model: 'gemini-2.5-pro', - timestamp: new Date(), processingTimeMs: 100, }, _internal: {}, }); - // Override sentence structure to return "Slightly Complex" + // Override sentence structure to return "Slightly complex" sentenceSpy.mockResolvedValue({ - score: 'Slightly Complex', + score: 'Slightly complex', reasoning: 'Sentence reasoning', metadata: { - promptVersion: '1.0', model: 'gpt-4o', - timestamp: new Date(), processingTimeMs: 100, }, _internal: {}, @@ -267,34 +252,36 @@ describe('TextComplexityEvaluator', () => { const result = await evaluator.evaluate(text, grade); - expect(result.score.vocabulary).toBe('moderately complex'); - expect(result.score.sentenceStructure).toBe('Slightly Complex'); + expect('error' in result.vocabulary).toBe(false); + expect('error' in result.sentenceStructure).toBe(false); + if (!('error' in result.vocabulary)) { + expect(result.vocabulary.score).toBe('Moderately complex'); + } + if (!('error' in result.sentenceStructure)) { + expect(result.sentenceStructure.score).toBe('Slightly complex'); + } }); - it('should build combined reasoning from both evaluators', async () => { + it('should preserve individual sub-evaluator reasoning', async () => { const text = 'The cat sat on the mat.'; const grade = '5'; // Override both evaluators with specific reasoning vocabSpy.mockResolvedValue({ - score: 'moderately complex', + score: 'Moderately complex', reasoning: 'This is the vocabulary reasoning.', metadata: { - promptVersion: '1.0', model: 'gemini-2.5-pro', - timestamp: new Date(), processingTimeMs: 100, }, _internal: {}, }); sentenceSpy.mockResolvedValue({ - score: 'Slightly Complex', + score: 'Slightly complex', reasoning: 'This is the sentence structure reasoning.', metadata: { - promptVersion: '1.0', model: 'gpt-4o', - timestamp: new Date(), processingTimeMs: 100, }, _internal: {}, @@ -302,10 +289,12 @@ describe('TextComplexityEvaluator', () => { const result = await evaluator.evaluate(text, grade); - expect(result.reasoning).toContain('Vocabulary Complexity'); - expect(result.reasoning).toContain('This is the vocabulary reasoning.'); - expect(result.reasoning).toContain('Sentence Structure Complexity'); - expect(result.reasoning).toContain('This is the sentence structure reasoning.'); + if (!('error' in result.vocabulary)) { + expect(result.vocabulary.reasoning).toBe('This is the vocabulary reasoning.'); + } + if (!('error' in result.sentenceStructure)) { + expect(result.sentenceStructure.reasoning).toBe('This is the sentence structure reasoning.'); + } }); }); diff --git a/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts b/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts index 9792acb..dee51e4 100644 --- a/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts +++ b/sdks/typescript/tests/unit/evaluators/vocabulary.test.ts @@ -96,7 +96,7 @@ describe('VocabularyEvaluator - Evaluation Flow', () => { // Mock complexity evaluation response vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ data: { - complexity_score: 'moderately complex', + complexity_score: 'Moderately complex', reasoning: 'The text uses grade-appropriate vocabulary.', factors: ['Academic terminology', 'Clear structure'], }, @@ -112,7 +112,7 @@ describe('VocabularyEvaluator - Evaluation Flow', () => { const result = await evaluator.evaluate(testText, testGrade); // Verify result structure - expect(result.score).toBe('moderately complex'); + expect(result.score).toBe('Moderately complex'); expect(result.reasoning).toContain('grade-appropriate vocabulary'); expect(result.metadata).toBeDefined(); expect(result.metadata.model).toBe('openai:gpt-4o-2024-11-20 + openai:gpt-4.1-2025-04-14'); @@ -190,7 +190,7 @@ describe('VocabularyEvaluator - Evaluation Flow', () => { vi.mocked(mockComplexityProvider.generateStructured).mockResolvedValue({ data: { - complexity_score: 'moderately complex', + complexity_score: 'Moderately complex', reasoning: 'Detailed reasoning here', factors: ['Factor 1', 'Factor 2'], }, @@ -208,15 +208,11 @@ describe('VocabularyEvaluator - Evaluation Flow', () => { expect(result).toHaveProperty('_internal'); // Verify metadata structure - expect(result.metadata).toHaveProperty('promptVersion'); expect(result.metadata).toHaveProperty('model'); - expect(result.metadata).toHaveProperty('timestamp'); expect(result.metadata).toHaveProperty('processingTimeMs'); // Verify metadata values - expect(result.metadata.promptVersion).toBe('1.2.0'); expect(result.metadata.model).toBe('openai:gpt-4o-2024-11-20 + openai:gpt-4.1-2025-04-14'); - expect(result.metadata.timestamp).toBeInstanceOf(Date); expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); // Mocked calls can be instant (0ms) }); @@ -228,7 +224,7 @@ describe('VocabularyEvaluator - Evaluation Flow', () => { }); const mockComplexityData = { - complexity_score: 'moderately complex', + complexity_score: 'Moderately complex', reasoning: 'Detailed reasoning', factors: ['Factor 1', 'Factor 2'], analysis: 'Deep analysis', From 871b07b292a293de1c62cd47c279acd10d59c268 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Fri, 13 Mar 2026 12:36:26 -0700 Subject: [PATCH 9/9] Implement Changelog --- sdks/typescript/CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 sdks/typescript/CHANGELOG.md diff --git a/sdks/typescript/CHANGELOG.md b/sdks/typescript/CHANGELOG.md new file mode 100644 index 0000000..516fb03 --- /dev/null +++ b/sdks/typescript/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +All notable changes to the `@learning-commons/evaluators` TypeScript SDK will be documented in this file. + +## [0.1.0] — Early Release + +Initial early release of the TypeScript SDK for Learning Commons educational evaluators. + +### Added + +- **Vocabulary Evaluator** — grades 3–12 vocabulary difficulty assessment. +- **Sentence Structure Evaluator** — syntactic complexity analysis by grade level. +- **Grade Level Appropriateness (GLA) Evaluator** — overall grade-level suitability scoring. +- **Text Complexity Evaluator** — composite evaluation combining Vocabulary, Sentence Structure, and GLA. +- **Provider abstraction** — model-agnostic via Vercel AI SDK; OpenAI, Google, and Anthropic supported. +- **Telemetry** — opt-in, with `partnerKey` and `recordInputs` (defaults to `false`). +- **Prompt versioning** — prompts versioned in `evals/prompts/` (v1.2.0), shared with Python notebooks.