diff --git a/toloka/README.md b/toloka/README.md new file mode 100644 index 0000000..56d7437 --- /dev/null +++ b/toloka/README.md @@ -0,0 +1,20 @@ +# Yandex.Toloka assessment system for human-based evaluation + +We use human-based evaluation for ELSA. + +## Interface + + + + + +## Settings + +## Questions + +## Instruction + +## Data + +## Results + diff --git a/toloka/interface/interface.css b/toloka/interface/interface.css new file mode 100644 index 0000000..06367f8 --- /dev/null +++ b/toloka/interface/interface.css @@ -0,0 +1,85 @@ +.fix_height { + max-height: calc(400px); + display: inline-block; +} + +.fix_width { + max-width: calc(45%); + display: inline-block; +} + +.left { + vertical-align: top; +} + +.page { + overflow-y: scroll !important; + margin-left: 30px; + line-height: 25px; +} + +.content { + display: inline-block; + text-align: justify; + margin-left: 10px; + margin-right: 10px; + margin-bottom: 20px; +} + +.right { + margin-top: 25px; + margin-left: 30px; + line-height: 25px; +} + +.clearfix { + overflow: hidden; + width: 100%; +} + +.question { + font-weight: 600; +} + +.question-grid { + display: inline; + margin-top: 50px; +} + +.question-item { + max-width: 45%; + display: inline-block; + text-align: left; + margin-bottom: 20px; + margin-left: 0px; +} + +.question-item-last { + text-align: left; + margin-bottom: 20px; +} + +.question-item-penult { + max-width: 45%; + display: inline-block; + text-align: left; + margin-bottom: 20px; + float: right; +} + +.source-text-title { + font-weight: 600; + margin-left: 20px; +} + +.button { + margin: 10px; + max-width: 182px; +} + +.button__label { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 150px; +} diff --git a/toloka/interface/interface.html b/toloka/interface/interface.html new file mode 100644 index 0000000..7508fce --- /dev/null +++ b/toloka/interface/interface.html @@ -0,0 +1,97 @@ +
+
+

Article

+

{{source_text}}

+

Summary #1

+

{{human_based_summary}}

+

Summary #2

+

{{machine_based_summary}}


+ + {{button label="To questions!" href=url_right size="L"}} +
+ +
+
+
+ + Q1. Does the summary #1 retain the main idea of the source text? +
+ {{field type="radio" name="s1-main-idea" label="Yes" value=true}}
+ {{field type="radio" name="s1-main-idea" label="No" value=false}}
+
+
+ + Q2. Does the summary #2 retain the main idea of the source text? +
+ {{field type="radio" name="s2-main-idea" label="Yes" value=true}}
+ {{field type="radio" name="s2-main-idea" label="No" value=false}}
+
+
+ + Q3. Does the summary #1 syntatically and grammatically coherent? +
+ {{field type="radio" name="s1-coherence" label="Yes" value=true}}
+ {{field type="radio" name="s1-coherence" label="No" value=false}}
+
+
+ + Q4. Is the summary #2 syntatically and grammatically coherent? +
+ {{field type="radio" name="s2-coherence" label="Yes" value=true}}
+ {{field type="radio" name="s2-coherence" label="No" value=false}}
+
+
+ + Q5. Does the summary #1 distort any facts present in the source text? +
+ {{field type="radio" name="s1-factual-misrepresentation" label="Yes" value=true}}
+ {{field type="radio" name="s1-factual-misrepresentation" label="No" value=false}}
+
+
+ + Q6. Does the summary #2 distort any facts present in the source text? +
+ {{field type="radio" name="s2-factual-misrepresentation" label="Yes" value=true}}
+ {{field type="radio" name="s2-factual-misrepresentation" label="No" value=false}}
+
+
+ + Q7. Does summary #1 contain sentences identical to those found in the source text? +
+ {{field type="radio" name="s1-identical-sentences" label="Yes" value=true}}
+ {{field type="radio" name="s1-identical-sentences" label="No" value=false}}
+
+
+ + Q8. Does summary #2 contain sentences identical to those found in the source text? +
+ {{field type="radio" name="s2-identical-sentences" label="Yes" value=true}}
+ {{field type="radio" name="s2-identical-sentences" label="No" value=false}}
+
+
+ + Q9. Would you be able to deduce that a human did not write summary #1? +
+ {{field type="radio" name="s1-machine" label="Yes" value=true}}
+ {{field type="radio" name="s1-machine" label="No" value=false}}
+
+
+ + Q10. Would you be able to deduce that a human did not write summary #2? +
+ {{field type="radio" name="s2-machine" label="Yes" value=true}}
+ {{field type="radio" name="s2-machine" label="No" value=false}}
+
+
+ + Q11. On a scale of 1 to 5 (5 being nearly identical), how closely does summary #2 resemble summary #1? +
+ {{field type="radio" name="resemblance" label="1" value=1}} + {{field type="radio" name="resemblance" label="2" value=2}} + {{field type="radio" name="resemblance" label="3" value=3}} + {{field type="radio" name="resemblance" label="4" value=4}} + {{field type="radio" name="resemblance" label="5" value=5}} +
+
+
+
diff --git a/toloka/interface/interface.js b/toloka/interface/interface.js new file mode 100644 index 0000000..8a56393 --- /dev/null +++ b/toloka/interface/interface.js @@ -0,0 +1,28 @@ +exports.Task = extend(TolokaHandlebarsTask, function (options) { + TolokaHandlebarsTask.call(this, options); +}, { + onRender: function() { + const task = this.getDOMElement(); + task.querySelector('.right').style.display = 'none'; + + task.querySelector('button').onclick = function(){ + task.querySelector('button').style.display = 'none'; + task.querySelector('.right').style.display = 'inline-block'; + }; + }, + onDestroy: function() { + // Задание завершено, можно освобождать (если были использованы) глобальные ресурсы + } +}); + +function extend(ParentClass, constructorFunction, prototypeHash) { + constructorFunction = constructorFunction || function () {}; + prototypeHash = prototypeHash || {}; + if (ParentClass) { + constructorFunction.prototype = Object.create(ParentClass.prototype); + } + for (var i in prototypeHash) { + constructorFunction.prototype[i] = prototypeHash[i]; + } + return constructorFunction; +}