ERASER scripts for hatexplain in /scripts folder

frontend/graphedit.py

@@ -0,0 +1,120 @@
+import argparse
+import copy
+import os
+import time
+import json
+import streamlit as st
+import pandas as pd
+import penman
+
+from graphviz import Source
+from xpotato.dataset.utils import default_pn_to_graph
+
+from tuw_nlp.graph.utils import graph_to_pn, pn_to_graph
+
+from utils import (
+    train_df,
+    add_rule_manually,
+    annotate_df,
+    extract_data_from_dataframe,
+    get_df_from_rules,
+    graph_viewer,
+    init_evaluator,
+    init_extractor,
+    init_session_states,
+    rank_and_suggest,
+    read_df,
+    rerun,
+    rule_chooser,
+    save_ruleset,
+    read_ruleset,
+    save_after_modify,
+    save_dataframe,
+    match_texts,
+    show_ml_feature,
+    st_stdout,
+    to_dot,
+)
+
+def main():
+    st.set_page_config(layout="wide")
+    hide_streamlit_style = """
+            <style>
+            #MainMenu {visibility: hidden;}
+            footer {
+
+                visibility: hidden;
+
+                }
+            footer:after {
+                content:'GraphEdit - mx.markus.rei@gmx.net'; 
+                visibility: visible;
+                display: block;
+                position: relative;
+                #background-color: red;
+                padding: 5px;
+                top: 2px;
+            }
+            </style>
+
+            """
+    st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+    st.markdown(
+        "<h1 style='text-align: center; color: black;'>GraphEdit</h1>"
+        "<h2 style='text-align: center; color: black;'>Edit ud PENMAN graphs!</h2>",
+        unsafe_allow_html=True,
+    )
+    init_session_states()
+    text_input = st.text_area("Provide the text here you want to convert to Penman notation!"
+    )
+    topenman = st.button("To Penman!")
+    output = st.empty()
+    penman_input = st.text_area("Provide the penman notation here you want to render!"
+    )
+    global generatedpenman
+    render = st.button("Render!")
+    extractor = init_extractor("en", "ud")
+    if topenman:
+    	if text_input:
+    		texts = text_input.split("\n")
+    		graphs = list(extractor.parse_iterable([text for text in texts], "ud"))
+    		dot_current_graph = to_dot(
+    		graphs[0],
+    		)
+    		penmanstring = value=(penman.encode(penman.decode(graph_to_pn(graphs[0])), indent=10))
+    		#penman_input = placeholder.text_area("Penman notation provided.", penmanstring)
+    		output.text(penmanstring)
+    		generatedpenman = penmanstring
+    if render:
+    	#penman_input = placeholder.text_area("Provide the penman notation here you want to render!", penmanstring)
+    	if penman_input:
+
+    		graph, ind = default_pn_to_graph(penman_input)
+    		dot_current_graph = to_dot(
+    		graph,
+    		)
+
+    		if st.session_state.download:
+    			graph_pipe = Source(dot_current_graph).pipe(format="svg")
+    			st.download_button(
+    			label="Download graph as SVG",
+    			data=graph_pipe,
+    			file_name="graph.svg",
+    			mime="mage/svg+xml",
+    			)
+
+    		with st.expander("Graph dot source", expanded=False):
+    			st.write(dot_current_graph)
+
+    		st.graphviz_chart(
+    		dot_current_graph,
+    		use_container_width=True,
+    		)
+
+    		st.write("Penman format:")
+    		st.text(penman.encode(penman.decode(graph_to_pn(graph)), indent=10))
+    		st.write("In one line format:")
+    		st.write(graph_to_pn(graph))
+
+if __name__ == "__main__":
+    main()

scripts/auto_evaluate.sh

@@ -0,0 +1,47 @@
+# potato
+function evaluate() {
+	path=$1
+	rules=$2
+
+	# process path to folder
+	foldersign="/"
+	minus="-"
+	foldername=${path//$foldersign/$minus}
+	tsv=".tsv"
+	empty=""
+	foldername=${foldername//$tsv/$empty}
+	foldername="$foldername-rules"
+
+	mkdir $foldername
+	cp $path val.tsv
+	python evaluate_hatexplain.py -f $rules -t val.tsv | tee evaluate.txt
+
+	cd eraserbenchmark
+	python print_eraser.py $foldername | tee out.txt
+	cd ..
+
+	cp temp_matched_result.tsv $foldername
+	cp temp_df_without_rationales.tsv $foldername
+	cp temp_df_only_rationales.tsv $foldername
+	mv evaluate.txt $foldername
+	mv eraserbenchmark/out.txt $foldername
+	mv cat_stats.json $foldername
+	mv eraser_output.json $foldername
+	rm val.tsv
+}
+
+evaluate women/minority_val_all.tsv sexism_rules.json
+evaluate women/majority_val_all.tsv sexism_rules.json
+evaluate homosexual/minority_val_all.tsv homophobia_rules.json
+evaluate homosexual/majority_val_all.tsv homophobia_rules.json
+
+evaluate women/minority_val_one_majority.tsv sexism_rules.json
+evaluate women/majority_val_one_majority.tsv sexism_rules.json
+evaluate homosexual/minority_val_one_majority.tsv homophobia_rules.json
+evaluate homosexual/majority_val_one_majority.tsv homophobia_rules.json
+
+evaluate women/minority_val_pure.tsv sexism_rules.json
+evaluate women/majority_val_pure.tsv sexism_rules.json
+evaluate homosexual/minority_val_pure.tsv homophobia_rules.json
+evaluate homosexual/majority_val_pure.tsv homophobia_rules.json
+

scripts/call_eraser.py

@@ -0,0 +1,93 @@
+import os, sys
+file_path = 'eraserbenchmark/'
+sys.path.append(os.path.dirname(file_path))
+
+import eraserbenchmark.rationale_benchmark.metrics as eb
+
+import contextlib, logging
+
+class DiscardEraserBenchMarkStdOut(object):
+    def write(self, x): pass
+
+@contextlib.contextmanager
+def nostdout():
+    save_stdout = sys.stdout
+    sys.stdout = DiscardEraserBenchMarkStdOut()
+    yield
+    sys.stdout = save_stdout
+
+#--data_dir : Location of the folder which contains the dataset in eraser format
+#--results : The location of the model output file in eraser format
+#--score_file : The file name and location to write the output
+
+def call_eraser(neutralclassname, datadir, testtrainorval, pathtopredictions, silent=False):
+	import sys, os
+	pkgpath = os.getcwd()+"\\eraserbenchmark"
+	print(pkgpath)
+	sys.path.append(pkgpath)
+	import rationale_benchmark.metrics as eraser
+	if silent:
+		logger = logging.getLogger()
+		logger.disabled = True
+	#dir(rationale_benchmark.metrics)
+	eraser.runEvaluation(neutralclassname, # neutralclassname
+						data_dir=datadir, # data dir
+						split=testtrainorval, # split
+						results=pathtopredictions, # results 
+						score_file=""+"./eraser_output.json", # score
+						strict=False) # strict
+						#iou_thresholds=[0.5], # iou
+						#aopc_thresholds=[0.01, 0.05, 0.1, 0.2, 0.5]) # aopc
+	if silent:
+		logger.disabled = False
+	print_eraser_results(".")
+
+"""
+def call_eraser(datadir, testtrainorval, pathtopredictions):
+	#args = ['--split', 'test', '--strict', '--data_dir', 'movies', '--results', './movies/movies_majority_human_perf.jsonl']
+	args = ['--split', testtrainorval, '--data_dir', datadir, '--results', pathtopredictions, '--score_file', 'eraser_output.json']
+	for arg in args:
+		sys.argv.append(arg)
+
+	# suppress text
+	SUPPRESS_TEXT = False
+
+	if SUPPRESS_TEXT:
+		logger = logging.getLogger()
+		logger.disabled = True
+		with nostdout():
+			eb.main()
+		logger.disabled = False
+	else:
+		eb.main()
+	print_eraser_results()
+"""
+
+def print_eraser_results(datadir):
+	# print the required results
+	import json
+	with open(datadir+'/eraser_output.json') as fp:
+		output_data = json.load(fp)
+
+	print("\n------------------------")
+
+	print('Plausibility')
+	if 'iou_scores' in output_data:
+		print('IOU F1 :', round(output_data['iou_scores'][0]['macro']['f1'], 3))
+		print('( P :', round(output_data['iou_scores'][0]['macro']['p'], 3), ', R :', round(output_data['iou_scores'][0]['macro']['r'], 3), ')')
+		print('Token F1 :', round(output_data['token_prf']['instance_macro']['f1'], 3))
+		print('( P :', round(output_data['token_prf']['instance_macro']['p'], 3), ', R :', round(output_data['token_prf']['instance_macro']['r'], 3), ')')
+
+	if 'token_soft_metrics' in output_data:
+		print('AUPRC :', round(output_data['token_soft_metrics']['auprc'], 3))
+
+	print('\nFaithfulness')
+	if 'classification_scores' in output_data:
+		print('Comprehensiveness :', round(output_data['classification_scores']['comprehensiveness'], 3))
+		print('Sufficiency :', round(output_data['classification_scores']['sufficiency'], 3))
+	else:
+		print('--')
+	print("")
+
+if __name__ == "__main__":
+	call_eraser("None", "./hatexplain", "val", "./hatexplain/val_prediction.jsonl")

scripts/eraserbenchmark/.gitignore

@@ -0,0 +1,111 @@
+model_components
+pipeline_outputs
+*.swp
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+rationale_benchmark/data/esnli_previous
+data/esnli_previous
+esnli_union/