From 7285e5db16ec1a53c5f96cdd8c2eef300dc2ec2f Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 22 Sep 2021 23:47:26 -0400 Subject: [PATCH 1/7] Added forward on instance methods to base model --- .../models/base.py | 64 +++++++++++++++++++ .../feedforward_score.py | 2 +- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/structured_prediction_baselines/models/base.py b/structured_prediction_baselines/models/base.py index 0cf44d19..b5b28656 100644 --- a/structured_prediction_baselines/models/base.py +++ b/structured_prediction_baselines/models/base.py @@ -10,7 +10,10 @@ Generator, ) import contextlib + +import numpy import torch +from allennlp.data import Instance, Batch from allennlp.models import Model from structured_prediction_baselines.modules.sampler import ( Sampler, @@ -404,3 +407,64 @@ def forward_on_scorenn( results["loss"] = loss return results + + def forward_on_instance(self, instance: Instance, mode: Optional[ModelMode] = None) -> Dict[str, numpy.ndarray]: + """ + Takes an [`Instance`](../data/instance.md), which typically has raw text in it, converts + that text into arrays using this model's [`Vocabulary`](../data/vocabulary.md), passes those + arrays through `self.forward()` and `self.make_output_human_readable()` (which by default + does nothing) and returns the result. Before returning the result, we convert any + `torch.Tensors` into numpy arrays and remove the batch dimension. + """ + return self.forward_on_instances([instance], mode)[0] + + def forward_on_instances( + self, + instances: List[Instance], + mode: Optional[ModelMode] = None + ) -> List[Dict[str, numpy.ndarray]]: + """ + Takes a list of `Instances`, converts that text into arrays using this model's `Vocabulary`, + passes those arrays through `self.forward()` and `self.make_output_human_readable()` (which + by default does nothing) and returns the result. Before returning the result, we convert + any `torch.Tensors` into numpy arrays and separate the batched output into a list of + individual dicts per instance. Note that typically this will be faster on a GPU (and + conditionally, on a CPU) than repeated calls to `forward_on_instance`. + + # Parameters + + instances : `List[Instance]`, required + The instances to run the model on. + + # Returns + + A list of the models output for each instance. + """ + batch_size = len(instances) + with torch.no_grad(): + cuda_device = self._get_prediction_device() + dataset = Batch(instances) + dataset.index_instances(self.vocab) + model_input = util.move_to_device(dataset.as_tensor_dict(), cuda_device) + outputs = self.make_output_human_readable(self(**model_input, mode=mode)) + + instance_separated_output: List[Dict[str, numpy.ndarray]] = [ + {} for _ in dataset.instances + ] + for name, output in list(outputs.items()): + if isinstance(output, torch.Tensor): + # NOTE(markn): This is a hack because 0-dim pytorch tensors are not iterable. + # This occurs with batch size 1, because we still want to include the loss in that case. + if output.dim() == 0: + output = output.unsqueeze(0) + + if output.size(0) != batch_size: + self._maybe_warn_for_unseparable_batches(name) + continue + output = output.detach().cpu().numpy() + elif len(output) != batch_size: + self._maybe_warn_for_unseparable_batches(name) + continue + for instance_output, batch_element in zip(instance_separated_output, output): + instance_output[name] = batch_element + return instance_separated_output diff --git a/structured_prediction_baselines/modules/structured_score/multilabel_classification/feedforward_score.py b/structured_prediction_baselines/modules/structured_score/multilabel_classification/feedforward_score.py index 0a61c53c..de93152d 100644 --- a/structured_prediction_baselines/modules/structured_score/multilabel_classification/feedforward_score.py +++ b/structured_prediction_baselines/modules/structured_score/multilabel_classification/feedforward_score.py @@ -27,7 +27,7 @@ def forward( buffer: Dict, **kwargs: Any, ) -> torch.Tensor: - hidden = self.feedforward(y) # (batch, num_samples, hidden_dim) + hidden = self.feedforward(y.float()) # (batch, num_samples, hidden_dim) score = torch.nn.functional.linear( hidden, self.projection_vector ) # unormalized (batch, num_samples) From f6442bc3415f862b5d710a5775d4628d59269901 Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Thu, 23 Sep 2021 00:08:46 -0400 Subject: [PATCH 2/7] Added base for plotting contour plots --- analysis/Contour Plots.py | 151 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 analysis/Contour Plots.py diff --git a/analysis/Contour Plots.py b/analysis/Contour Plots.py new file mode 100644 index 00000000..75b6b4c6 --- /dev/null +++ b/analysis/Contour Plots.py @@ -0,0 +1,151 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.12.0 +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% +import os, sys + +path = os.path.dirname(os.path.abspath('.')) +if not path in sys.path: sys.path.append(path) + +# %% +import torch +from allennlp.models import Model +from allennlp.common import Params +from allennlp.common import plugins +from allennlp.common import util as common_util +common_util.import_module_and_submodules('structured_prediction_baselines') + +# %% +from allennlp.data import ( + Field, + DataLoader, + DatasetReader, + Instance, + Batch, + Vocabulary, + TextFieldTensors, +) +from allennlp.training import Trainer +from allennlp.nn import util + +from structured_prediction_baselines.common import ModelMode + +# %% [markdown] +# # Load model, dataset-reader, data and vocab + +# %% +serialization_dir = "../model_wts/" # path to the directory containing config, model weights and vocab +config_file = os.path.join(serialization_dir, "config.json") +vocabulary_dir = os.path.join(serialization_dir, "vocabulary") +weights_file = os.path.join(serialization_dir, "weights.th") + +# %% +loaded_params = Params.from_file(config_file) +loaded_model = Model.load(loaded_params, serialization_dir, weights_file) +loaded_model.eval() + +dataset_reader = DatasetReader.from_params(loaded_params.get("dataset_reader")) + +data_loader_params = loaded_params.get("data_loader") +train_data_loader = DataLoader.from_params( + params=data_loader_params.duplicate(), + reader=dataset_reader, + data_path='../' + loaded_params.get("train_data_path") +) + +dev_data_loader = DataLoader.from_params( + reader=dataset_reader, + data_path='../' + loaded_params.get("validation_data_path"), + params=data_loader_params, +) + +vocab = Vocabulary.from_instances(train_data_loader.iter_instances()) + +# %% +train_data_loader.index_with(vocab) +dev_data_loader.index_with(vocab) + +# %% +# uncomment if you want to load trainer +# loaded_params['trainer']['cuda_device'] = -1 +# loaded_params['trainer']['num_epochs'] = 1 + +# trainer = Trainer.from_params( +# model=loaded_model, +# serialization_dir=serialization_dir, +# data_loader=train_data_loader, +# validation_data_loader=dev_data_loader, +# params=loaded_params.pop("trainer"), +# ) + +# %% [markdown] +# ## Get an instance from the data + +# %% +train_instances = train_data_loader.iter_instances() +inst0 = next(train_instances) +instances = [inst0] + +# %% +loaded_model.forward_on_instance(inst0, mode=ModelMode.COMPUTE_SCORE) + +# %% +device = 0 if torch.cuda.is_available() else -1 +with torch.no_grad(): + dataset = Batch(instances) + dataset.index_instances(vocab) + model_input = util.move_to_device(dataset.as_tensor_dict(), device=device) + +# %% +model_input + +# %% +with torch.no_grad(): + model_output = loaded_model(model_input['x'], model_input['labels'], mode=ModelMode.COMPUTE_SCORE) + +# %% +model_output + +# %% [markdown] +# # Contour Plots + +# %% +# %matplotlib inline +import matplotlib.pyplot as plt +plt.style.use('seaborn-white') +import numpy as np + +# %% +y0 = np.linspace(0, 1, 500) +y1 = np.linspace(0, 1, 500) +z = np.zeros((500, 500)) + +# %% +x = model_input['x'] +y = model_input['labels'].clone().detach().float() +for i in range(len(y0)): + for j in range(len(y1)): + y[0][0] = y0[i] + y[0][1] = y1[j] + with torch.no_grad(): + z[i][j] = loaded_model(x, y, mode=ModelMode.COMPUTE_SCORE)['score'] + +# %% +plt.contour(y1, y0, z, 20, colors='black', ) +plt.colorbar() +plt.xlabel("y1") +plt.ylabel("y0") +plt.show() + +# %% From 52409860cbdde751f761bc14b85c3e441da0779b Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Thu, 23 Sep 2021 01:17:30 -0400 Subject: [PATCH 3/7] rename notebook --- analysis/Contour Plots.py | 151 ------------------------------------ analysis/Visulatizations.py | 133 ++++++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 154 deletions(-) delete mode 100644 analysis/Contour Plots.py diff --git a/analysis/Contour Plots.py b/analysis/Contour Plots.py deleted file mode 100644 index 75b6b4c6..00000000 --- a/analysis/Contour Plots.py +++ /dev/null @@ -1,151 +0,0 @@ -# --- -# jupyter: -# jupytext: -# text_representation: -# extension: .py -# format_name: percent -# format_version: '1.3' -# jupytext_version: 1.12.0 -# kernelspec: -# display_name: Python 3 (ipykernel) -# language: python -# name: python3 -# --- - -# %% -import os, sys - -path = os.path.dirname(os.path.abspath('.')) -if not path in sys.path: sys.path.append(path) - -# %% -import torch -from allennlp.models import Model -from allennlp.common import Params -from allennlp.common import plugins -from allennlp.common import util as common_util -common_util.import_module_and_submodules('structured_prediction_baselines') - -# %% -from allennlp.data import ( - Field, - DataLoader, - DatasetReader, - Instance, - Batch, - Vocabulary, - TextFieldTensors, -) -from allennlp.training import Trainer -from allennlp.nn import util - -from structured_prediction_baselines.common import ModelMode - -# %% [markdown] -# # Load model, dataset-reader, data and vocab - -# %% -serialization_dir = "../model_wts/" # path to the directory containing config, model weights and vocab -config_file = os.path.join(serialization_dir, "config.json") -vocabulary_dir = os.path.join(serialization_dir, "vocabulary") -weights_file = os.path.join(serialization_dir, "weights.th") - -# %% -loaded_params = Params.from_file(config_file) -loaded_model = Model.load(loaded_params, serialization_dir, weights_file) -loaded_model.eval() - -dataset_reader = DatasetReader.from_params(loaded_params.get("dataset_reader")) - -data_loader_params = loaded_params.get("data_loader") -train_data_loader = DataLoader.from_params( - params=data_loader_params.duplicate(), - reader=dataset_reader, - data_path='../' + loaded_params.get("train_data_path") -) - -dev_data_loader = DataLoader.from_params( - reader=dataset_reader, - data_path='../' + loaded_params.get("validation_data_path"), - params=data_loader_params, -) - -vocab = Vocabulary.from_instances(train_data_loader.iter_instances()) - -# %% -train_data_loader.index_with(vocab) -dev_data_loader.index_with(vocab) - -# %% -# uncomment if you want to load trainer -# loaded_params['trainer']['cuda_device'] = -1 -# loaded_params['trainer']['num_epochs'] = 1 - -# trainer = Trainer.from_params( -# model=loaded_model, -# serialization_dir=serialization_dir, -# data_loader=train_data_loader, -# validation_data_loader=dev_data_loader, -# params=loaded_params.pop("trainer"), -# ) - -# %% [markdown] -# ## Get an instance from the data - -# %% -train_instances = train_data_loader.iter_instances() -inst0 = next(train_instances) -instances = [inst0] - -# %% -loaded_model.forward_on_instance(inst0, mode=ModelMode.COMPUTE_SCORE) - -# %% -device = 0 if torch.cuda.is_available() else -1 -with torch.no_grad(): - dataset = Batch(instances) - dataset.index_instances(vocab) - model_input = util.move_to_device(dataset.as_tensor_dict(), device=device) - -# %% -model_input - -# %% -with torch.no_grad(): - model_output = loaded_model(model_input['x'], model_input['labels'], mode=ModelMode.COMPUTE_SCORE) - -# %% -model_output - -# %% [markdown] -# # Contour Plots - -# %% -# %matplotlib inline -import matplotlib.pyplot as plt -plt.style.use('seaborn-white') -import numpy as np - -# %% -y0 = np.linspace(0, 1, 500) -y1 = np.linspace(0, 1, 500) -z = np.zeros((500, 500)) - -# %% -x = model_input['x'] -y = model_input['labels'].clone().detach().float() -for i in range(len(y0)): - for j in range(len(y1)): - y[0][0] = y0[i] - y[0][1] = y1[j] - with torch.no_grad(): - z[i][j] = loaded_model(x, y, mode=ModelMode.COMPUTE_SCORE)['score'] - -# %% -plt.contour(y1, y0, z, 20, colors='black', ) -plt.colorbar() -plt.xlabel("y1") -plt.ylabel("y0") -plt.show() - -# %% diff --git a/analysis/Visulatizations.py b/analysis/Visulatizations.py index 01ef8763..75b6b4c6 100644 --- a/analysis/Visulatizations.py +++ b/analysis/Visulatizations.py @@ -7,18 +7,145 @@ # format_version: '1.3' # jupytext_version: 1.12.0 # kernelspec: -# display_name: Python 3 +# display_name: Python 3 (ipykernel) # language: python # name: python3 # --- # %% -# Contains visulatizations for energy +import os, sys + +path = os.path.dirname(os.path.abspath('.')) +if not path in sys.path: sys.path.append(path) + +# %% +import torch +from allennlp.models import Model +from allennlp.common import Params +from allennlp.common import plugins +from allennlp.common import util as common_util +common_util.import_module_and_submodules('structured_prediction_baselines') + +# %% +from allennlp.data import ( + Field, + DataLoader, + DatasetReader, + Instance, + Batch, + Vocabulary, + TextFieldTensors, +) +from allennlp.training import Trainer +from allennlp.nn import util + +from structured_prediction_baselines.common import ModelMode + +# %% [markdown] +# # Load model, dataset-reader, data and vocab + +# %% +serialization_dir = "../model_wts/" # path to the directory containing config, model weights and vocab +config_file = os.path.join(serialization_dir, "config.json") +vocabulary_dir = os.path.join(serialization_dir, "vocabulary") +weights_file = os.path.join(serialization_dir, "weights.th") + +# %% +loaded_params = Params.from_file(config_file) +loaded_model = Model.load(loaded_params, serialization_dir, weights_file) +loaded_model.eval() + +dataset_reader = DatasetReader.from_params(loaded_params.get("dataset_reader")) + +data_loader_params = loaded_params.get("data_loader") +train_data_loader = DataLoader.from_params( + params=data_loader_params.duplicate(), + reader=dataset_reader, + data_path='../' + loaded_params.get("train_data_path") +) + +dev_data_loader = DataLoader.from_params( + reader=dataset_reader, + data_path='../' + loaded_params.get("validation_data_path"), + params=data_loader_params, +) + +vocab = Vocabulary.from_instances(train_data_loader.iter_instances()) + +# %% +train_data_loader.index_with(vocab) +dev_data_loader.index_with(vocab) + +# %% +# uncomment if you want to load trainer +# loaded_params['trainer']['cuda_device'] = -1 +# loaded_params['trainer']['num_epochs'] = 1 + +# trainer = Trainer.from_params( +# model=loaded_model, +# serialization_dir=serialization_dir, +# data_loader=train_data_loader, +# validation_data_loader=dev_data_loader, +# params=loaded_params.pop("trainer"), +# ) + +# %% [markdown] +# ## Get an instance from the data + +# %% +train_instances = train_data_loader.iter_instances() +inst0 = next(train_instances) +instances = [inst0] + +# %% +loaded_model.forward_on_instance(inst0, mode=ModelMode.COMPUTE_SCORE) + +# %% +device = 0 if torch.cuda.is_available() else -1 +with torch.no_grad(): + dataset = Batch(instances) + dataset.index_instances(vocab) + model_input = util.move_to_device(dataset.as_tensor_dict(), device=device) + +# %% +model_input # %% +with torch.no_grad(): + model_output = loaded_model(model_input['x'], model_input['labels'], mode=ModelMode.COMPUTE_SCORE) + +# %% +model_output + +# %% [markdown] +# # Contour Plots + +# %% +# %matplotlib inline import matplotlib.pyplot as plt +plt.style.use('seaborn-white') +import numpy as np + +# %% +y0 = np.linspace(0, 1, 500) +y1 = np.linspace(0, 1, 500) +z = np.zeros((500, 500)) + +# %% +x = model_input['x'] +y = model_input['labels'].clone().detach().float() +for i in range(len(y0)): + for j in range(len(y1)): + y[0][0] = y0[i] + y[0][1] = y1[j] + with torch.no_grad(): + z[i][j] = loaded_model(x, y, mode=ModelMode.COMPUTE_SCORE)['score'] # %% -plt.plot([1,2], [2,3]) +plt.contour(y1, y0, z, 20, colors='black', ) +plt.colorbar() +plt.xlabel("y1") +plt.ylabel("y0") +plt.show() # %% From f1058a4885a3455134b0532bb31b1ec30d856527 Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Thu, 23 Sep 2021 16:55:41 -0400 Subject: [PATCH 4/7] jupytext config change, contour plot dimension fix --- analysis/Visulatizations.py | 45 ++++++++++++++++++++++++------------- jupytext.toml | 5 ++--- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/analysis/Visulatizations.py b/analysis/Visulatizations.py index 75b6b4c6..5315b748 100644 --- a/analysis/Visulatizations.py +++ b/analysis/Visulatizations.py @@ -72,6 +72,9 @@ vocab = Vocabulary.from_instances(train_data_loader.iter_instances()) +# %% +loaded_model + # %% train_data_loader.index_with(vocab) dev_data_loader.index_with(vocab) @@ -93,8 +96,8 @@ # ## Get an instance from the data # %% -train_instances = train_data_loader.iter_instances() -inst0 = next(train_instances) +dev_instances = dev_data_loader.iter_instances() +inst0 = next(dev_instances) instances = [inst0] # %% @@ -126,26 +129,36 @@ plt.style.use('seaborn-white') import numpy as np + # %% -y0 = np.linspace(0, 1, 500) -y1 = np.linspace(0, 1, 500) -z = np.zeros((500, 500)) +def compute_score_values(model, model_input, dim1, dim2, rmin=-0.5, rmax=1.5, n=100, sigmoid=False): + dim1_values = np.linspace(rmin, rmax, n) + dim2_values = np.linspace(rmin, rmax, n) + dim1_mesh, dim2_mesh = np.meshgrid(dim1_values, dim2_values) + z = np.zeros((n, n)) + + x = model_input['x'] + y = model_input['labels'].clone().detach().float() + # y = 1 - y + for i in range(len(dim1_values)): + for j in range(len(dim2_values)): + y[0][dim1] = dim1_mesh[i][j] + y[0][dim2] = dim2_mesh[i][j] + with torch.no_grad(): + z[j][i] = loaded_model(x, y, mode=ModelMode.COMPUTE_SCORE)['score'] + if sigmoid: + z = torch.sigmoid(torch.from_numpy(z)).numpy() + return dim1_mesh, dim2_mesh, z + # %% -x = model_input['x'] -y = model_input['labels'].clone().detach().float() -for i in range(len(y0)): - for j in range(len(y1)): - y[0][0] = y0[i] - y[0][1] = y1[j] - with torch.no_grad(): - z[i][j] = loaded_model(x, y, mode=ModelMode.COMPUTE_SCORE)['score'] +Y0, Y1, z = compute_score_values(loaded_model, model_input, 0, -1, n=200, sigmoid=True) # %% -plt.contour(y1, y0, z, 20, colors='black', ) +plt.contour(Y0, Y1, z, 20, cmap='RdGy') plt.colorbar() -plt.xlabel("y1") -plt.ylabel("y0") +plt.xlabel("y0") +plt.ylabel("y1") plt.show() # %% diff --git a/jupytext.toml b/jupytext.toml index 4aab4e1f..39811ea0 100644 --- a/jupytext.toml +++ b/jupytext.toml @@ -1,3 +1,2 @@ -[formats] -"notebooks/" = "ipynb" -"analysis/" = "py:percent" +formats = "notebooks///ipynb,analysis///py:percent" +outdated_text_notebook_margin = 5.0 \ No newline at end of file From e7566d57121417e4dae2fed35937eebf33154fed Mon Sep 17 00:00:00 2001 From: Dhruvesh Date: Mon, 7 Feb 2022 18:51:08 -0500 Subject: [PATCH 5/7] Added code used for ICML plots --- analysis/Visulatizations.py | 254 +++++++++++++++++++++++++++++++++--- 1 file changed, 236 insertions(+), 18 deletions(-) diff --git a/analysis/Visulatizations.py b/analysis/Visulatizations.py index 5315b748..92342fed 100644 --- a/analysis/Visulatizations.py +++ b/analysis/Visulatizations.py @@ -7,14 +7,13 @@ # format_version: '1.3' # jupytext_version: 1.12.0 # kernelspec: -# display_name: Python 3 (ipykernel) +# display_name: Python 3 # language: python # name: python3 # --- # %% import os, sys - path = os.path.dirname(os.path.abspath('.')) if not path in sys.path: sys.path.append(path) @@ -22,7 +21,6 @@ import torch from allennlp.models import Model from allennlp.common import Params -from allennlp.common import plugins from allennlp.common import util as common_util common_util.import_module_and_submodules('structured_prediction_baselines') @@ -45,10 +43,10 @@ # # Load model, dataset-reader, data and vocab # %% -serialization_dir = "../model_wts/" # path to the directory containing config, model weights and vocab +serialization_dir = "../.allennlp_models/dvn_model_wts/expr_fun_dvn/" # path to the directory containing config, model weights and vocab config_file = os.path.join(serialization_dir, "config.json") vocabulary_dir = os.path.join(serialization_dir, "vocabulary") -weights_file = os.path.join(serialization_dir, "weights.th") +weights_file = os.path.join(serialization_dir, "best.th") # %% loaded_params = Params.from_file(config_file) @@ -70,7 +68,8 @@ params=data_loader_params, ) -vocab = Vocabulary.from_instances(train_data_loader.iter_instances()) +#vocab = Vocabulary.from_instances(train_data_loader.iter_instances()) +vocab = Vocabulary.from_files(vocabulary_dir) # %% loaded_model @@ -113,25 +112,21 @@ # %% model_input -# %% -with torch.no_grad(): - model_output = loaded_model(model_input['x'], model_input['labels'], mode=ModelMode.COMPUTE_SCORE) - -# %% -model_output - # %% [markdown] # # Contour Plots # %% # %matplotlib inline +# #%matplotlib notebook import matplotlib.pyplot as plt plt.style.use('seaborn-white') import numpy as np +import matplotlib # %% def compute_score_values(model, model_input, dim1, dim2, rmin=-0.5, rmax=1.5, n=100, sigmoid=False): + "Compute scores on a 2D mesh of (yi, yj) with other yk values and the input x fixed." dim1_values = np.linspace(rmin, rmax, n) dim2_values = np.linspace(rmin, rmax, n) dim1_mesh, dim2_mesh = np.meshgrid(dim1_values, dim2_values) @@ -152,13 +147,236 @@ def compute_score_values(model, model_input, dim1, dim2, rmin=-0.5, rmax=1.5, n= # %% -Y0, Y1, z = compute_score_values(loaded_model, model_input, 0, -1, n=200, sigmoid=True) +def compute_global_score_values(model, dim1, dim2, rmin=-0.5, rmax=1.5, n=100, sigmoid=False): + """Same as 'compute_score_values' but only uses global energy and hence does not take x.""" + dim1_values = np.linspace(rmin, rmax, n) + dim2_values = np.linspace(rmin, rmax, n) + dim1_mesh, dim2_mesh = np.meshgrid(dim1_values, dim2_values) + z = np.zeros((n, n)) + + y = torch.rand(1,loaded_model.vocab.get_vocab_size('labels')) + # y = 1 - y + for i in range(len(dim1_values)): + for j in range(len(dim2_values)): + y[0][dim1] = dim1_mesh[i][j] + y[0][dim2] = dim2_mesh[i][j] + with torch.no_grad(): + z[j][i] = loaded_model.score_nn.compute_global_score(y, buffer={}) + if sigmoid: + z = torch.sigmoid(torch.from_numpy(z)).numpy() + return dim1_mesh, dim2_mesh, z + + +# %% +Y0, Y1, z = compute_global_score_values(loaded_model, 46, 280, rmin=-0.5, rmax=1.5, n=200, sigmoid=False) + +# %% +# How to get indices for various labels? Uncomment the following code and make appropriate changes. +#print(vocab.get_token_index('18.01', namespace='labels')) +#print(vocab.get_token_index('18', namespace='labels')) + # %% -plt.contour(Y0, Y1, z, 20, cmap='RdGy') -plt.colorbar() -plt.xlabel("y0") -plt.ylabel("y1") +def compute_grad_2d(model, dim1, dim2, inner_loop=100, outer_loop=100): + x = [] + y = [] + z = [] + for i in range(outer_loop): + Y = torch.rand(1,loaded_model.vocab.get_vocab_size('labels')) + Y.requires_grad = True + x_inner = [] + y_inner = [] + z_inner = [] + for j in range(inner_loop): + v1 = np.random.rand(1) + v2 = np.random.rand(1) + Y[0][dim1] = float(v1) + Y[0][dim2] = float(v2) + Z = torch.autograd.grad(loaded_model.score_nn.compute_global_score(Y, buffer={}), Y)[0][0][dim2] + x_inner.append(float(Y[0][dim1])) + y_inner.append(float(Y[0][dim2])) + z_inner.append(float(Z)) + x+=x_inner + y+=y_inner + #z+=z_inner + #z+=((np.array(z_inner) - np.mean(z_inner))/np.mean(z_inner)).tolist() + #z+=((np.array(z_inner))/np.sum(z_inner)).tolist() + temp = np.array(z_inner) + z += ((temp - temp.mean())/temp.std()).tolist() + + return x, y, z + + +# %% +#plt.rcParams['text.usetex'] = True +x2,y2,z2 = compute_grad_2d(loaded_model, 55, 45, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +plt.colorbar(sc) +#plt.xlabel('y_i') +#plt.ylabel('y_j') +plt.show() + +# %% +yk = 45 +yi = 55 +# yi \implies yk +x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel.pdf') plt.show() # %% +x2,y2,z2 = compute_grad_2d(loaded_model, 21, 55, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_no_rel.pdf') + +plt.show() + +# %% +x2 = np.random.rand(1000) +y2 = np.random.rand(1000) +temp = 1/y2 +z2 = (temp -temp.mean())/temp.std() +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm, vmax=1) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_ce.pdf') + +plt.show() + +# %% +x1,y1,z1 = compute_grad_2d(loaded_model, 55, 45, inner_loop=100, outer_loop=10,) +x2,y2,z2 = compute_grad_2d(loaded_model, 21, 55, inner_loop=100, outer_loop=10,) +x3 = np.random.rand(1000) +y3 = np.random.rand(1000) +temp = 1/y3 +z3 = (temp -temp.mean())/temp.std() +fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(10,3), constrained_layout=True) +cm = plt.cm.get_cmap('RdYlBu') +axs[0].scatter(x3, y3, c=z3, cmap=cm) +axs[1].scatter(x1, y1, c=z1, cmap=cm) +axs[2].scatter(x2, y2, c=z2, cmap=cm) +axs[0].set_ylabel(r"$y_k$", fontsize=15) +axs[1].set_xlabel(r"$y_i$", fontsize=15) +cb = plt.colorbar(sc,shrink=0.7, aspect=40*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.show() +fig.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads.pdf') + +# %% +print(w) + +# %% +#plt.rcParams['text.usetex'] = True +x2,y2,z2 = compute_grad_2d(loaded_model, 55, 45, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +plt.colorbar(sc) +#plt.xlabel('y_i') +#plt.ylabel('y_j') +plt.show() + +# %% +yk = 15 +yi = 1 +# yi \implies yk +x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel_dvn_1_55.pdf') +plt.show() + +# %% +yk = 2 +yi = 1 +# yi \implies yk +x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel_dvn_1_55.pdf') +plt.show() + +# %% +yk = 26 +yi = 125 +# yi \implies yk +x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel_dvn_1_55.pdf') +plt.show() + +# %% +print(vocab.get_token_index('02.01', namespace='labels')) +print(vocab.get_token_index('02.01.01', namespace='labels')) + +print(vocab.get_token_from_index(2, namespace='labels')) +print(vocab.get_token_from_index(7, namespace='labels')) + +# %% +yk = 2 +yi = 7 +# yi \implies yk +x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel_dvn_1_55.pdf') +plt.show() + +# %% +yk = 125 +yi = 487 +# yi \implies yk +x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel_dvn_1_55.pdf') +plt.show() + +# %% +yk = 125 +yi = 35 +# yi \implies yk +x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) +cm = plt.cm.get_cmap('RdYlBu') +sc = plt.scatter(x2, y2, c=z2, cmap=cm) +cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) +cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) +plt.ylabel(r"$y_k$", fontsize=15) +plt.xlabel(r"$y_i$", fontsize=15) +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel_dvn_1_55.pdf') +plt.show() From 3ffc2026e9b59205c9cbffc935bc0d642dc869f6 Mon Sep 17 00:00:00 2001 From: Dhruvesh Date: Mon, 7 Feb 2022 18:55:01 -0500 Subject: [PATCH 6/7] Added code used for significance tests for icml --- analysis/significance-test-icml.py | 113 +++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 analysis/significance-test-icml.py diff --git a/analysis/significance-test-icml.py b/analysis/significance-test-icml.py new file mode 100644 index 00000000..a38c56cb --- /dev/null +++ b/analysis/significance-test-icml.py @@ -0,0 +1,113 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.12.0 +# kernelspec: +# display_name: Python 3 +# language: python +# name: python3 +# --- + +# %% +import pandas as pd +import numpy as np +from scipy import stats +import Orange +import matplotlib.pyplot as plt +#https://github.com/EGiunchiglia/C-HMCNN/blob/master/friedman_test.py + +# %% +csv_path = '/Users/dhruveshpatel/Downloads/scorenn-paper-assets/for-stat-test.csv' +multiplication_factor = -1 #-1 for rank 1-6 decreasing order +# subset_list = ['MBM-T', 'MHM-T', 'C-HMCNN'] +# subset_list = ['MVM', 'MHM', 'MBM', 'C-HMCNN', 'Box-E'] +subset_list = [ + 'CE', + 'SPEN', + 'DVN', + 'NCE ranking', + 'SEAL static margin', + 'SEAL static regression', + 'SEAL static NCE ranking', + 'SEAL dynamic margin', + 'SEAL dynamic regression', + 'SEAL dynamic regression-s', + 'SEAL dynamic NCE ranking'] + +# %% +data = pd.read_csv(csv_path) +data + +# %% +data = data[['Dataset']+subset_list] +data + +# %% +model_names = data.columns[1:].to_list() +model_names + +# %% +measurements = (data[data.columns[1:]].to_numpy().T)*multiplication_factor +measurements + +# %% +rank_data = stats.rankdata(measurements, axis=0) +rank_data + +# %% + +avranks = np.mean(rank_data, axis=1) +print(list(zip(model_names, avranks))) +avranks + +# %% +array_ranks = [list(rank_data[i, :]) for i in range(rank_data.shape[0])] +print(*array_ranks) + +# %% +stats.friedmanchisquare(*array_ranks) + +# %% +avranks = list(avranks) + +# %% +cd = Orange.evaluation.compute_CD(avranks, 7) +print(cd) +Orange.evaluation.graph_ranks(avranks, model_names, cd=cd, width=6, textspace=1.5, alpha=0.05) +plt.title(f"Based on results across datasets") +plt.savefig(f"nemenyi.pdf", format='pdf' ,bbox_inches='tight',pad_inches=0) + +# %% +data + +# %% +model_names + +# %% +for model1 in ['CE', 'SEAL dynamic NCE ranking']: + for model2 in model_names: + if model1 == model2: + continue + print(f"Wilcoxon between {model1} and {model2}") + print(stats.wilcoxon(data[model1], data[model2], alternative="less")) + +# %% +for model1 in ['SEAL static NCE ranking']: + for model2 in model_names: + if model1 == model2: + continue + print(f"Wilcoxon between {model1} and {model2}") + print(stats.wilcoxon(data[model1], data[model2], alternative="less")) + +# %% +data['SEAL dynamic NCE ranking'] + +# %% + +# %% + +# %% From 0c7b694fe876f77304efb705cb8791a044dc9272 Mon Sep 17 00:00:00 2001 From: Dhruvesh Date: Sun, 13 Feb 2022 13:33:46 -0500 Subject: [PATCH 7/7] Update visuazliation to plot 6 plots --- analysis/Visulatizations.py | 71 ++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/analysis/Visulatizations.py b/analysis/Visulatizations.py index 92342fed..5bf08d76 100644 --- a/analysis/Visulatizations.py +++ b/analysis/Visulatizations.py @@ -19,6 +19,7 @@ # %% import torch +from pathlib import Path from allennlp.models import Model from allennlp.common import Params from allennlp.common import util as common_util @@ -43,6 +44,7 @@ # # Load model, dataset-reader, data and vocab # %% +dataset_name = 'expr_fun' serialization_dir = "../.allennlp_models/dvn_model_wts/expr_fun_dvn/" # path to the directory containing config, model weights and vocab config_file = os.path.join(serialization_dir, "config.json") vocabulary_dir = os.path.join(serialization_dir, "vocabulary") @@ -170,11 +172,6 @@ def compute_global_score_values(model, dim1, dim2, rmin=-0.5, rmax=1.5, n=100, s # %% Y0, Y1, z = compute_global_score_values(loaded_model, 46, 280, rmin=-0.5, rmax=1.5, n=200, sigmoid=False) -# %% -# How to get indices for various labels? Uncomment the following code and make appropriate changes. -#print(vocab.get_token_index('18.01', namespace='labels')) -#print(vocab.get_token_index('18', namespace='labels')) - # %% def compute_grad_2d(model, dim1, dim2, inner_loop=100, outer_loop=100): @@ -207,8 +204,66 @@ def compute_grad_2d(model, dim1, dim2, inner_loop=100, outer_loop=100): return x, y, z +# %% +# How to get indices for various labels? Uncomment the following code and make appropriate changes. +# Related +related_indices = [(vocab.get_token_index(yi, namespace='labels'), vocab.get_token_index(yk, namespace='labels')) for yi, yk in [ + ('18.01', '18'), + ('01.01', '01'), + ('18.02.01', '18.02'), + ('16.01', '16'), + ('10.01', '10'), + ('01.05', '01')]] +print(related_indices) + + +# %% +# Generate and save plots +base_path = Path('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/') +for yi, yk in related_indices: + # yi \implies yk + x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) + cm = plt.cm.get_cmap('RdYlBu') + sc = plt.scatter(x2, y2, c=z2, cmap=cm) + cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) + cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) + plt.ylabel(r"$y_k$", fontsize=15) + plt.xlabel(r"$y_i$", fontsize=15) + plt.savefig(base_path / f'grads_positive_rel_{dataset_name}_{yi}_{yk}.pdf') + plt.show() + +# %% +# Generate and save plots +# How to get indices for various labels? Uncomment the following code and make appropriate changes. +# Related +un_related_indices = [(vocab.get_token_index(yi, namespace='labels'), vocab.get_token_index(yk, namespace='labels')) for yi, yk in [ + ('32.01', '30.01'), + ('20.01', '01.01.03.05'), + ('01.01.03.05', '18.02.01.01'), + ('01.01.03.05', '18.02'), + ('01.01.03.05', '16.21.15'), + ('30.05', '01.01.03.05') + ]] +print(un_related_indices) +base_path = Path('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/') +for yi, yk in un_related_indices: + # yi \implies yk + x2,y2,z2 = compute_grad_2d(loaded_model, yk , yi, inner_loop=100, outer_loop=10,) + cm = plt.cm.get_cmap('RdYlBu') + sc = plt.scatter(x2, y2, c=z2, cmap=cm) + cb = plt.colorbar(sc,shrink=0.7, aspect=20*0.7) + cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) + plt.ylabel(r"$y_k$", fontsize=15) + plt.xlabel(r"$y_i$", fontsize=15) + plt.savefig(base_path / f'grads_positive_no_rel_{dataset_name}_{yi}_{yk}.pdf') + plt.show() + +# %% [markdown] +# # Rough work + # %% #plt.rcParams['text.usetex'] = True + x2,y2,z2 = compute_grad_2d(loaded_model, 55, 45, inner_loop=100, outer_loop=10,) cm = plt.cm.get_cmap('RdYlBu') sc = plt.scatter(x2, y2, c=z2, cmap=cm) @@ -228,7 +283,7 @@ def compute_grad_2d(model, dim1, dim2, inner_loop=100, outer_loop=100): cb.ax.set_title(r"$\frac{\partial E_\Theta^{g}}{\partial y_k}$", fontsize=20) plt.ylabel(r"$y_k$", fontsize=15) plt.xlabel(r"$y_i$", fontsize=15) -plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel.pdf') +plt.savefig('/Users/dhruveshpatel/Downloads/scorenn-paper-assets/grads_positive_rel_{}.pdf') plt.show() # %% @@ -336,8 +391,8 @@ def compute_grad_2d(model, dim1, dim2, inner_loop=100, outer_loop=100): print(vocab.get_token_index('02.01', namespace='labels')) print(vocab.get_token_index('02.01.01', namespace='labels')) -print(vocab.get_token_from_index(2, namespace='labels')) -print(vocab.get_token_from_index(7, namespace='labels')) +print(vocab.get_token_from_index(21, namespace='labels')) +print(vocab.get_token_from_index(55, namespace='labels')) # %% yk = 2