From 50ff77832f662694123f3b0d338394a2e93856f8 Mon Sep 17 00:00:00 2001 From: leejayyoon Date: Sat, 6 Nov 2021 16:09:35 -0400 Subject: [PATCH 01/18] added model, sweep configs for the NER-SEAL_NCE runs --- ...rt_adapter_nce_bert_adapter_tasknn.jsonnet | 223 ++++++++++++++++++ ..._bert_adapter_seal_nce_linearchain.jsonnet | 172 ++++++++++++++ .../conll2003_bert_adapter_tasknn.jsonnet | 137 +++++++++++ ...conll2003_seal_nce_cnnscore_bilstm.jsonnet | 198 ++++++++++++++++ ...ll2003_seal_nce_linearchain_bilstm.jsonnet | 198 ++++++++++++++++ .../conll2003_tasknn_bilstm.jsonnet | 187 +++++++++++++++ .../gendata_seal_nce_bilstm.jsonnet | 198 ++++++++++++++++ .../conll2003_bilstm_tasknn.yaml | 25 ++ .../conll2003_nce_discrete_cnn_score.yaml | 47 ++++ ...ll2003_nce_discrete_linearchain_score.yaml | 47 ++++ ...ata_nce_discrete_minus_tasknn_reverse.yaml | 49 ++++ 11 files changed, 1481 insertions(+) create mode 100644 model_configs/sequence_tagging/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet create mode 100644 model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet create mode 100644 model_configs/sequence_tagging/conll2003_bert_adapter_tasknn.jsonnet create mode 100644 model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet create mode 100644 model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet create mode 100644 model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet create mode 100644 model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet create mode 100644 sweep_configs/sequence_tagging/conll2003_bilstm_tasknn.yaml create mode 100644 sweep_configs/sequence_tagging/conll2003_nce_discrete_cnn_score.yaml create mode 100644 sweep_configs/sequence_tagging/conll2003_nce_discrete_linearchain_score.yaml create mode 100644 sweep_configs/sequence_tagging/gendata_nce_discrete_minus_tasknn_reverse.yaml diff --git a/model_configs/sequence_tagging/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet b/model_configs/sequence_tagging/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..a4e7f8c1 --- /dev/null +++ b/model_configs/sequence_tagging/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet @@ -0,0 +1,223 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'bgc'; +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-basic', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-score-loss', + log_key: 'neg_nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-nce-ranking-with-discrete-sampling', + log_key: 'nce', + num_samples: 100, + sign: '-', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + //num_gradient_accumulation_steps: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet new file mode 100644 index 00000000..7672798f --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet @@ -0,0 +1,172 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +local max_length = 512; + +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_mismatched_with_adapter', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'pretrained_transformer_mismatched', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 64, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_tasknn.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..aa82a443 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_tasknn.jsonnet @@ -0,0 +1,137 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +local max_length = 512; + +local ff_activation = 'softplus'; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_mismatched_with_adapter', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'pretrained_transformer_mismatched', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'mean', // mean will work fine because seq-tagging-masked-ce will take care of masking + normalize_y: false, // don't normalize because ce requires logits + }, + }, + loss_fn: { + type: 'zero', // there is no score_nn so we put a dummy zero loss + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet new file mode 100644 index 00000000..b66c37df --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet @@ -0,0 +1,198 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +//local ff_dropout = std.parseJson(std.extVar('ff_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'sequence-tagging-with-infnet', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'cnn', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet new file mode 100644 index 00000000..ce66a0b1 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet @@ -0,0 +1,198 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +//local ff_dropout = std.parseJson(std.extVar('ff_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'sequence-tagging-with-infnet', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet new file mode 100644 index 00000000..03e8f0bc --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet @@ -0,0 +1,187 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +//local ff_dropout = std.parseJson(std.extVar('ff_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local inference_score_weight = 1; //std.parseJson(std.extVar('inference_score_weight')); +local cross_entropy_loss_weight = 1; //std.parseJson(std.extVar('cross_entropy_loss_weight')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, + +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'sequence-tagging-with-infnet', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'sequence-tagging-margin-based', + reduction: 'mean', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + log_key: 'margin_loss' +}, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} \ No newline at end of file diff --git a/model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet b/model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet new file mode 100644 index 00000000..9d4b88ae --- /dev/null +++ b/model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet @@ -0,0 +1,198 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +//local ff_dropout = std.parseJson(std.extVar('ff_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: dataset_name, + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'sequence-tagging-with-infnet', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_bilstm_tasknn.yaml b/sweep_configs/sequence_tagging/conll2003_bilstm_tasknn.yaml new file mode 100644 index 00000000..7e9f41f4 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_bilstm_tasknn.yaml @@ -0,0 +1,25 @@ +name: conll2003_bilstm_tasknn +description: "Train tasknn using cross-entropy." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_nce_discrete_cnn_score.yaml b/sweep_configs/sequence_tagging/conll2003_nce_discrete_cnn_score.yaml new file mode 100644 index 00000000..33b7834c --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_nce_discrete_cnn_score.yaml @@ -0,0 +1,47 @@ +name: conll2003_nce_discrete_minus_tasknn_reverse +description: "Train tasknn using cross-entropy and score loss (v(f(x),y)). The score-nn will be trained using NCE with a - sign (score-ln Pn). The samples are taken as discrete samples from the tasknn output." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.dvn_score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + # distribution: q_uniform + # min: 0 + # max: 11.99 + # q: 3 + model.loss_fn.num_samples: + value: 100 + # distribution: q_uniform + # q: 20 # 10, 25, ..., 50 + # min: 20 # ln(10) + # max: 100 # ln(50) \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_nce_discrete_linearchain_score.yaml b/sweep_configs/sequence_tagging/conll2003_nce_discrete_linearchain_score.yaml new file mode 100644 index 00000000..e194e229 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_nce_discrete_linearchain_score.yaml @@ -0,0 +1,47 @@ +name: conll2003_nce_discrete_minus_tasknn_reverse +description: "Train tasknn using cross-entropy and score loss (v(f(x),y)). The score-nn will be trained using NCE with a - sign (score-ln Pn). The samples are taken as discrete samples from the tasknn output." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/conll2003_seal_nce_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.dvn_score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + # distribution: q_uniform + # min: 0 + # max: 11.99 + # q: 3 + model.loss_fn.num_samples: + value: 100 + # distribution: q_uniform + # q: 20 # 10, 25, ..., 50 + # min: 20 # ln(10) + # max: 100 # ln(50) \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/gendata_nce_discrete_minus_tasknn_reverse.yaml b/sweep_configs/sequence_tagging/gendata_nce_discrete_minus_tasknn_reverse.yaml new file mode 100644 index 00000000..2303d2db --- /dev/null +++ b/sweep_configs/sequence_tagging/gendata_nce_discrete_minus_tasknn_reverse.yaml @@ -0,0 +1,49 @@ +name: [data_name]_nce_discrete_minus_tasknn_reverse +description: "Train tasknn using cross-entropy and score loss (v(f(x),y)). The score-nn will be trained using NCE with a - sign (score-ln Pn). The samples are taken as discrete samples from the tasknn output." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=[data_name],inference_module=inference_net,inference_module=tasknn" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.dataset_name: + value: '[data_name]' + env.cross_entropy_loss_weight: + value: 1.0 + env.dvn_score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + # distribution: q_uniform + # min: 0 + # max: 11.99 + # q: 3 + model.loss_fn.num_samples: + value: 100 + # distribution: q_uniform + # q: 20 # 10, 25, ..., 50 + # min: 20 # ln(10) + # max: 100 # ln(50) \ No newline at end of file From c758a457df09a56f530d4cffba75ecb458bc52dd Mon Sep 17 00:00:00 2001 From: leejayyoon Date: Mon, 8 Nov 2021 17:49:47 -0500 Subject: [PATCH 02/18] adding model configs for running SEAL-NCE with NER (conll2003) --- ...onll2003_bert_adapter_seal_nce_cnn.jsonnet | 172 ++++++++++++++++++ ..._bert_adapter_seal_nce_linearchain.jsonnet | 3 +- .../conll2003_inference_net_stacked.jsonnet | 2 +- ...conll2003_seal_nce_cnnscore_bilstm.jsonnet | 2 +- ...ll2003_seal_nce_linearchain_bilstm.jsonnet | 2 +- .../conll2003_tasknn_bilstm.jsonnet | 2 +- .../conll2003_tasknn_ff.jsonnet | 2 +- 7 files changed, 178 insertions(+), 7 deletions(-) create mode 100644 model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet new file mode 100644 index 00000000..d1420f80 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet @@ -0,0 +1,172 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +local max_length = 512; + +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_mismatched_with_adapter', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'pretrained_transformer_mismatched', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'cnn', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 64, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet index 7672798f..f2774d9c 100644 --- a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet @@ -14,7 +14,6 @@ local ff_activation = 'softplus'; local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); local weight_decay = std.parseJson(std.extVar('weight_decay')); -local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); @@ -138,7 +137,7 @@ local task_nn = { optimizers: { task_nn: { - lr: tasknn_lr, + lr: 0.00001, weight_decay: weight_decay, type: 'huggingface_adamw', }, diff --git a/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet b/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet index 8b2401d1..2c277cac 100644 --- a/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet +++ b/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet @@ -63,7 +63,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { - type: 'sequence-tagging-with-infnet', + type: 'seal-ner', label_encoding: 'BIOUL', sampler: { type: 'appending-container', diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet index b66c37df..75780c57 100644 --- a/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet +++ b/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet @@ -84,7 +84,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { - type: 'sequence-tagging-with-infnet', + type: 'seal-ner', label_encoding: 'BIOUL', sampler: { type: 'appending-container', diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet index ce66a0b1..c14f0d34 100644 --- a/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet +++ b/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet @@ -84,7 +84,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { - type: 'sequence-tagging-with-infnet', + type: 'seal-ner', label_encoding: 'BIOUL', sampler: { type: 'appending-container', diff --git a/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet index 03e8f0bc..bba9a5fe 100644 --- a/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet +++ b/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet @@ -83,7 +83,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { - type: 'sequence-tagging-with-infnet', + type: 'seal-ner', label_encoding: 'BIOUL', sampler: { type: 'appending-container', diff --git a/model_configs/sequence_tagging/conll2003_tasknn_ff.jsonnet b/model_configs/sequence_tagging/conll2003_tasknn_ff.jsonnet index 59e7ccd5..46c4e6b2 100644 --- a/model_configs/sequence_tagging/conll2003_tasknn_ff.jsonnet +++ b/model_configs/sequence_tagging/conll2003_tasknn_ff.jsonnet @@ -65,7 +65,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { - type: 'sequence-tagging-with-infnet', + type: 'seal-ner', label_encoding: 'BIOUL', sampler: { type: 'appending-container', From b3b8973ee4cab5c2df3d524221777147d5f46c16 Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 10 Nov 2021 11:58:38 +0530 Subject: [PATCH 03/18] Added ner seal nce self_attention config --- ...003_seal_nce_self_attention_bilstm.jsonnet | 202 ++++++++++++++++++ ...ll2003_seal_nce_self_attention_bilstm.yaml | 48 +++++ 2 files changed, 250 insertions(+) create mode 100644 model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet create mode 100644 sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.yaml diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet new file mode 100644 index 00000000..58ac84d9 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet @@ -0,0 +1,202 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.yaml new file mode 100644 index 00000000..a55d4104 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.yaml @@ -0,0 +1,48 @@ +name: conll2003_seal_nce_discrete_self_attention_bilstm +description: "Train tasknn using cross-entropy and score loss (v(f(x),y)). The score-nn will be trained using NCE with a - sign (score-ln Pn). The samples are taken as discrete samples from the tasknn output." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.dvn_score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + model.loss_fn.num_samples: + value: 100 + env.attention_dim: + distribution: q_uniform + min: 49 # 50 + max: 249 # 200 + q: 50 + env.attention_dropout_10x: + distribution: q_uniform + min: 0.5 # 1 + max: 5.49 # 5 \ No newline at end of file From 640930b3a665a50a4099ca308afaa67eb8e9bbf9 Mon Sep 17 00:00:00 2001 From: leejayyoon Date: Fri, 3 Dec 2021 19:15:08 -0500 Subject: [PATCH 04/18] jsonnet additions --- .../v2.5/bgc_bert_adapter_dvn.jsonnet | 240 +++++++++++++++++ ...pter_margin_l1_bert_adapter_tasknn.jsonnet | 241 +++++++++++++++++ ...rt_adapter_nce_bert_adapter_tasknn.jsonnet | 5 +- .../v2.5/gendata_bert_adapter_dvn.jsonnet | 236 +++++++++++++++++ ...pter_margin_l1_bert_adapter_tasknn.jsonnet | 242 +++++++++++++++++ .../v2.5/nyt_bert_adapter_dvn.jsonnet | 240 +++++++++++++++++ ...pter_margin_l1_bert_adapter_tasknn.jsonnet | 238 +++++++++++++++++ .../v2.5/rcv_bert_adapter_dvn.jsonnet | 240 +++++++++++++++++ ...pter_margin_l1_bert_adapter_tasknn.jsonnet | 238 +++++++++++++++++ .../v2.5/rcv_bert_adapter_spen.jsonnet | 247 ++++++++++++++++++ .../v2.5/rcv_bert_nce_bert_tasknn.jsonnet} | 27 +- ...onll2003_bert_adapter_seal_nce_cnn.jsonnet | 4 +- ..._bert_adapter_seal_nce_linearchain.jsonnet | 4 +- ...conll2003_seal_nce_cnnscore_bilstm.jsonnet | 2 +- .../conll2003_tasknn_bilstm.jsonnet | 4 - ...ntonotes_bert_adapter_seal_nce_cnn.jsonnet | 166 ++++++++++++ ..._bert_adapter_seal_nce_linearchain.jsonnet | 166 ++++++++++++ .../srl/ontonotes_seal_nce_bert.jsonnet | 172 ++++++++++++ 18 files changed, 2691 insertions(+), 21 deletions(-) create mode 100644 model_configs/multilabel_classification/v2.5/bgc_bert_adapter_dvn.jsonnet create mode 100644 model_configs/multilabel_classification/v2.5/bgc_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet create mode 100644 model_configs/multilabel_classification/v2.5/gendata_bert_adapter_dvn.jsonnet create mode 100644 model_configs/multilabel_classification/v2.5/gendata_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet create mode 100644 model_configs/multilabel_classification/v2.5/nyt_bert_adapter_dvn.jsonnet create mode 100644 model_configs/multilabel_classification/v2.5/nyt_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet create mode 100644 model_configs/multilabel_classification/v2.5/rcv_bert_adapter_dvn.jsonnet create mode 100644 model_configs/multilabel_classification/v2.5/rcv_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet create mode 100644 model_configs/multilabel_classification/v2.5/rcv_bert_adapter_spen.jsonnet rename model_configs/{sequence_tagging/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet => multilabel_classification/v2.5/rcv_bert_nce_bert_tasknn.jsonnet} (89%) create mode 100644 model_configs/srl/ontonotes_bert_adapter_seal_nce_cnn.jsonnet create mode 100644 model_configs/srl/ontonotes_bert_adapter_seal_nce_linearchain.jsonnet create mode 100644 model_configs/srl/ontonotes_seal_nce_bert.jsonnet diff --git a/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_dvn.jsonnet b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_dvn.jsonnet new file mode 100644 index 00000000..d3d573cc --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_dvn.jsonnet @@ -0,0 +1,240 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'bgc'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [ + //GBI + { + type: 'gradient-based-inference', + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss // change this + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + // Adversarial + { + type: 'gradient-based-inference', + log_key: 'adv', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'negative', + log_key: 'neg', + constituent_loss: { type: 'multi-label-dvn-bce', reduction: 'none', log_key: 'dvn_bce' }, + reduction: 'none', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + + { type: 'ground-truth' }, + ], + }, + inference_module: { + type: 'gradient-based-inference', + log_key: 'inference', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 30, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { type: 'multi-label-dvn-bce', log_key: 'dvn_bce' }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..356f8e87 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet @@ -0,0 +1,241 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'bgc'; +local data_reader_name = (if std.toString(dataset_name) == 'nyt10' then 'nyt' else dataset_name); # nyt10 or bgc +local data_reader_name = (if std.toString(dataset_name) == 'rcv1' then 'rcv' else data_reader_name); # nyt10 or rcv1 + +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local inference_score_weight = std.parseJson(std.extVar('inference_score_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-inference-net-normalized', + log_key: 'inference_module', + cost_augmented_layer: { + type: 'multi-label-stacked', + feedforward: { + input_dim: 2 * num_labels, + num_layers: 2, + activations: [ff_activation, 'linear'], + hidden_dims: num_labels, + }, + normalize_y: true, + }, + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-inference', + log_key: 'neg_inference', + normalize_y: true, + reduction: 'none', + inference_score_weight: inference_score_weight, + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [1.0, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { + type: 'manhattan', + differentiable: true, + }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-margin-based', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + num_gradient_accumulation_steps: 2, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet index a4e7f8c1..995963dd 100644 --- a/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet +++ b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet @@ -30,6 +30,9 @@ local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_w local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); + + + local feature_network = { text_field_embedder: { token_embedders: { @@ -71,7 +74,6 @@ local feature_network = { model_name: transformer_model, max_length: 512, }, - }, train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + dataset_metadata.train_file), @@ -79,7 +81,6 @@ local feature_network = { dataset_metadata.validation_file), test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + dataset_metadata.test_file), - vocabulary: { type: 'from_files', directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' diff --git a/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_dvn.jsonnet b/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_dvn.jsonnet new file mode 100644 index 00000000..a86749d2 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_dvn.jsonnet @@ -0,0 +1,236 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [ + //GBI + { + type: 'gradient-based-inference', + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss // change this + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + // Adversarial + { + type: 'gradient-based-inference', + log_key: 'adv', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'negative', + log_key: 'neg', + constituent_loss: { type: 'multi-label-dvn-bce', reduction: 'none', log_key: 'dvn_bce' }, + reduction: 'none', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + + { type: 'ground-truth' }, + ], + }, + inference_module: { + type: 'gradient-based-inference', + log_key: 'inference', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 30, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { type: 'multi-label-dvn-bce', log_key: 'dvn_bce' }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + shuffle: true, + batch_size: 32, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..61396ee7 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet @@ -0,0 +1,242 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; +// local dataset_name = std.parseJson(std.extVar('dataset_name')); +local data_reader_name = (if std.toString(dataset_name) == 'nyt10' then 'nyt' else dataset_name); // nyt10 or bgc +local data_reader_name = (if std.toString(dataset_name) == 'rcv1' then 'rcv' else data_reader_name); // nyt10 or rcv1 + +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local inference_score_weight = std.parseJson(std.extVar('inference_score_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-inference-net-normalized', + log_key: 'inference_module', + cost_augmented_layer: { + type: 'multi-label-stacked', + feedforward: { + input_dim: 2 * num_labels, + num_layers: 2, + activations: [ff_activation, 'linear'], + hidden_dims: num_labels, + }, + normalize_y: true, + }, + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-inference', + log_key: 'neg_inference', + normalize_y: true, + reduction: 'none', + inference_score_weight: inference_score_weight, + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [1.0, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { + type: 'manhattan', + differentiable: true, + }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-margin-based', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + //num_gradient_accumulation_steps: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_dvn.jsonnet b/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_dvn.jsonnet new file mode 100644 index 00000000..2b2b8621 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_dvn.jsonnet @@ -0,0 +1,240 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'nyt10'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'nyt', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [ + //GBI + { + type: 'gradient-based-inference', + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss // change this + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + // Adversarial + { + type: 'gradient-based-inference', + log_key: 'adv', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'negative', + log_key: 'neg', + constituent_loss: { type: 'multi-label-dvn-bce', reduction: 'none', log_key: 'dvn_bce' }, + reduction: 'none', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + + { type: 'ground-truth' }, + ], + }, + inference_module: { + type: 'gradient-based-inference', + log_key: 'inference', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 30, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { type: 'multi-label-dvn-bce', log_key: 'dvn_bce' }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..9bd4434f --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet @@ -0,0 +1,238 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'nyt10'; +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local inference_score_weight = std.parseJson(std.extVar('inference_score_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'nyt', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-inference-net-normalized', + log_key: 'inference_module', + cost_augmented_layer: { + type: 'multi-label-stacked', + feedforward: { + input_dim: 2 * num_labels, + num_layers: 2, + activations: [ff_activation, 'linear'], + hidden_dims: num_labels, + }, + normalize_y: true, + }, + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-inference', + log_key: 'neg_inference', + normalize_y: true, + reduction: 'none', + inference_score_weight: inference_score_weight, + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [1.0, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { + type: 'manhattan', + differentiable: true, + }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-margin-based', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + //num_gradient_accumulation_steps: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_dvn.jsonnet b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_dvn.jsonnet new file mode 100644 index 00000000..9ee7b2a1 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_dvn.jsonnet @@ -0,0 +1,240 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'rcv', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [ + //GBI + { + type: 'gradient-based-inference', + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss // change this + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + // Adversarial + { + type: 'gradient-based-inference', + log_key: 'adv', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'negative', + log_key: 'neg', + constituent_loss: { type: 'multi-label-dvn-bce', reduction: 'none', log_key: 'dvn_bce' }, + reduction: 'none', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + + { type: 'ground-truth' }, + ], + }, + inference_module: { + type: 'gradient-based-inference', + log_key: 'inference', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 30, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { type: 'multi-label-dvn-bce', log_key: 'dvn_bce' }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..27810b78 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet @@ -0,0 +1,238 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local inference_score_weight = std.parseJson(std.extVar('inference_score_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'rcv', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-inference-net-normalized', + log_key: 'inference_module', + cost_augmented_layer: { + type: 'multi-label-stacked', + feedforward: { + input_dim: 2 * num_labels, + num_layers: 2, + activations: [ff_activation, 'linear'], + hidden_dims: num_labels, + }, + normalize_y: true, + }, + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-inference', + log_key: 'neg_inference', + normalize_y: true, + reduction: 'none', + inference_score_weight: inference_score_weight, + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [1.0, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { + type: 'manhattan', + differentiable: true, + }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-margin-based', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + num_gradient_accumulation_steps: 2, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_spen.jsonnet b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_spen.jsonnet new file mode 100644 index 00000000..4fa86801 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_spen.jsonnet @@ -0,0 +1,247 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'rcv', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + sampler: { + type: 'gradient-based-inference-tasknn-init', + log_key: 'sampler', + inference_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + gbi_sampler: //GBI + { + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'multi-label-inference', + oracle_cost_weight: 1.0, + inference_score_weight: 1, + log_key: 'margin_loss', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + }, + inference_module: { + type: 'gradient-based-inference-tasknn-init', + log_key: 'sampler', + inference_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + gbi_sampler: //GBI + { + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'multi-label-score-loss', + reduction: 'none', + log_key: 'score_loss', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-structured-svm', + oracle_cost_weight: 1.0, + perceptron_loss_weight: 1, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + num_gradient_accumulation_steps: 2, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/rcv_bert_nce_bert_tasknn.jsonnet similarity index 89% rename from model_configs/sequence_tagging/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet rename to model_configs/multilabel_classification/v2.5/rcv_bert_nce_bert_tasknn.jsonnet index a4e7f8c1..e92c4514 100644 --- a/model_configs/sequence_tagging/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet +++ b/model_configs/multilabel_classification/v2.5/rcv_bert_nce_bert_tasknn.jsonnet @@ -3,7 +3,7 @@ local data_dir = std.extVar('DATA_DIR'); local cuda_device = std.extVar('CUDA_DEVICE'); local use_wandb = (if test == '1' then false else true); -local dataset_name = 'bgc'; +local dataset_name = 'rcv1'; local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; local num_labels = dataset_metadata.num_labels; local num_input_features = dataset_metadata.input_features; @@ -12,29 +12,36 @@ local num_input_features = dataset_metadata.input_features; // // common local ff_activation = 'softplus'; local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); -local ff_linear_layers = 2; -local weight_decay = std.parseJson(std.extVar('weight_decay')); -local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; +local ff_linear_layers = 1; +//local weight_decay = std.parseJson(std.extVar('weight_decay')); +local weight_decay = 0.1; +//local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; +local dropout = 0.5; // // score_nn local transformer_model = 'bert-base-uncased'; // huggingface name of the model local transformer_dim = 768; local transformer_vocab_size = 30522; local score_nn_weight_decay = weight_decay; -local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +//local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local global_score_hidden_dim = 600; local score_nn_dropout = dropout; // // task_nn local task_nn_dropout = dropout; local task_nn_weight_decay = weight_decay; -local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +//local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local cross_entropy_loss_weight = 1; local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +//local score_loss_weight = 3; +local num_samples = std.parseJson(std.extVar('num_samples')); + local feature_network = { text_field_embedder: { token_embedders: { x: { - type: 'pretrained_transformer_with_adapter', + type: 'pretrained_transformer', model_name: transformer_model, }, }, @@ -47,7 +54,7 @@ local feature_network = { feedforward: { input_dim: transformer_dim, num_layers: ff_linear_layers, - activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + ['linear']), hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), }, @@ -58,7 +65,7 @@ local feature_network = { evaluate_on_test: true, // Data dataset_reader: { - type: 'bgc', + type: 'rcv', //[if test == '1' then 'max_instances']: 100, token_indexers: { x: { @@ -149,7 +156,7 @@ vocabulary: { loss_fn: { type: 'multi-label-nce-ranking-with-discrete-sampling', log_key: 'nce', - num_samples: 100, + num_samples: num_samples, sign: '-', }, initializer: { diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet index d1420f80..dbcdb582 100644 --- a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet @@ -98,7 +98,7 @@ local task_nn = { type: 'seqtag-nce-ranking-with-discrete-sampling', reduction: 'mean', log_key: 'seq_nce_loss', - num_samples: 10, + num_samples: 200, }, initializer: { regexes: [ @@ -111,7 +111,7 @@ local task_nn = { data_loader: { batch_sampler: { type: 'bucket', - batch_size: 64, // effective batch size = batch_size*num_gradient_accumulation_steps + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps sorting_keys: ['tokens'], }, num_workers: 5, diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet index f2774d9c..581921d4 100644 --- a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet @@ -97,7 +97,7 @@ local task_nn = { type: 'seqtag-nce-ranking-with-discrete-sampling', reduction: 'mean', log_key: 'seq_nce_loss', - num_samples: 10, + num_samples: 200, }, initializer: { regexes: [ @@ -110,7 +110,7 @@ local task_nn = { data_loader: { batch_sampler: { type: 'bucket', - batch_size: 64, // effective batch size = batch_size*num_gradient_accumulation_steps + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps sorting_keys: ['tokens'], }, num_workers: 5, diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet index 75780c57..3dd703e8 100644 --- a/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet +++ b/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet @@ -129,7 +129,7 @@ local task_nn = { type: 'seqtag-nce-ranking-with-discrete-sampling', reduction: 'mean', log_key: 'seq_nce_loss', - num_samples: 10, + num_samples: 200, }, initializer: { regexes: [ diff --git a/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet index d5b995f8..6aa2fac1 100644 --- a/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet +++ b/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet @@ -81,11 +81,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { -<<<<<<< HEAD type: 'seal-ner', -======= - type: 'sequence-tagging-with-infnet', ->>>>>>> fd90a955d9edae124992ebbba14c219abe92180b label_encoding: 'BIOUL', sampler: { type: 'appending-container', diff --git a/model_configs/srl/ontonotes_bert_adapter_seal_nce_cnn.jsonnet b/model_configs/srl/ontonotes_bert_adapter_seal_nce_cnn.jsonnet new file mode 100644 index 00000000..23cee3ff --- /dev/null +++ b/model_configs/srl/ontonotes_bert_adapter_seal_nce_cnn.jsonnet @@ -0,0 +1,166 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); + + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_with_adapter', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'cnn', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 200, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: weight_decay, + type: 'adamw', + }, + + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/srl/ontonotes_bert_adapter_seal_nce_linearchain.jsonnet b/model_configs/srl/ontonotes_bert_adapter_seal_nce_linearchain.jsonnet new file mode 100644 index 00000000..c6f573f5 --- /dev/null +++ b/model_configs/srl/ontonotes_bert_adapter_seal_nce_linearchain.jsonnet @@ -0,0 +1,166 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); + + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_with_adapter', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 200, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: weight_decay, + type: 'adamw', + }, + + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/srl/ontonotes_seal_nce_bert.jsonnet b/model_configs/srl/ontonotes_seal_nce_bert.jsonnet new file mode 100644 index 00000000..1fb513fb --- /dev/null +++ b/model_configs/srl/ontonotes_seal_nce_bert.jsonnet @@ -0,0 +1,172 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local ff_weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 10, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} From be4cbd28980f07ee121e28720d2ad366abba46b3 Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 29 Dec 2021 14:47:43 -0500 Subject: [PATCH 05/18] Adds residual_x to seq-tag scorenn and corresponding parameter to control it, adds parameters to control cnn structured score --- .../conll2003_inference_net_stacked.jsonnet | 2 ++ .../modules/cnn_encoder.py | 8 ++++++ .../modules/score_nn.py | 2 ++ .../modules/sequence_tagging_score_nn.py | 7 ++++- .../modules/sequence_tagging_task_nn.py | 2 +- .../structured_score/sequence_tagging/cnn.py | 27 ++++++++++++++----- .../sequence_tagging/self_attention.py | 14 +++++++--- 7 files changed, 50 insertions(+), 12 deletions(-) diff --git a/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet b/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet index a88ddde5..ed694616 100644 --- a/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet +++ b/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet @@ -114,9 +114,11 @@ local task_nn = { score_nn: { type: 'sequence-tagging', task_nn: task_nn, + residual_x: true, global_score: { type: 'self-attention-full-sequence', num_tags: num_labels, + input_dim: num_labels + transformer_hidden_dim, num_heads: 1, attention_dim: 300, output_dim: 100, diff --git a/structured_prediction_baselines/modules/cnn_encoder.py b/structured_prediction_baselines/modules/cnn_encoder.py index 288f5c4d..9f58be52 100644 --- a/structured_prediction_baselines/modules/cnn_encoder.py +++ b/structured_prediction_baselines/modules/cnn_encoder.py @@ -62,6 +62,13 @@ def __init__( self.add_module("conv_layer_%d" % i, conv_layer) self._dropout = Dropout(dropout) + # TODO:: Fix maxpool_output_dim, Add pooling? + if output_dim: + self.projection_layer = Linear(maxpool_output_dim, output_dim) + self._output_dim = output_dim + else: + self.projection_layer = None + self._output_dim = maxpool_output_dim @overrides def get_input_dim(self) -> int: @@ -73,6 +80,7 @@ def get_output_dim(self) -> int: def forward(self, tokens: torch.Tensor, mask: torch.BoolTensor = None): if mask is not None: + # mask: (batch, seq_len) tokens = tokens * mask.unsqueeze(1).unsqueeze(-1) batch_size, n_samples, seq_length, _ = tokens.shape diff --git a/structured_prediction_baselines/modules/score_nn.py b/structured_prediction_baselines/modules/score_nn.py index 341d776a..615d5de6 100644 --- a/structured_prediction_baselines/modules/score_nn.py +++ b/structured_prediction_baselines/modules/score_nn.py @@ -12,10 +12,12 @@ def __init__( self, task_nn: TaskNN, # (batch, ...) global_score: Optional[StructuredScore] = None, + residual_x: bool = False, **kwargs: Any, ): super().__init__() # type:ignore self.task_nn = task_nn + self.residual_x = residual_x self.global_score = global_score self._dtype = self.compute_input_dtype() diff --git a/structured_prediction_baselines/modules/sequence_tagging_score_nn.py b/structured_prediction_baselines/modules/sequence_tagging_score_nn.py index 2680b33d..d12ddb98 100644 --- a/structured_prediction_baselines/modules/sequence_tagging_score_nn.py +++ b/structured_prediction_baselines/modules/sequence_tagging_score_nn.py @@ -54,7 +54,12 @@ def forward( score = None local_score = self.compute_local_score(x, y, buffer=buffer) - + if self.residual_x: + _, n_samples, _, _ = y.shape + embedded_x: torch.Tensor = buffer["embedded_x"] + assert len(embedded_x.shape) == 3 + embedded_x = embedded_x.unsqueeze(dim=1).repeat(1, n_samples, 1, 1) + y = torch.cat([y, embedded_x], dim=-1) global_score = self.compute_global_score(y, buffer) return local_score + global_score diff --git a/structured_prediction_baselines/modules/sequence_tagging_task_nn.py b/structured_prediction_baselines/modules/sequence_tagging_task_nn.py index 1cb7f7ca..4a5670ef 100644 --- a/structured_prediction_baselines/modules/sequence_tagging_task_nn.py +++ b/structured_prediction_baselines/modules/sequence_tagging_task_nn.py @@ -71,7 +71,7 @@ def forward( buffer["mask"] = mask embedded_text_input = self.text_field_embedder(tokens) - + buffer["embedded_x"] = embedded_text_input if self.encoder: encoded_text = self.encoder(embedded_text_input, mask) else: diff --git a/structured_prediction_baselines/modules/structured_score/sequence_tagging/cnn.py b/structured_prediction_baselines/modules/structured_score/sequence_tagging/cnn.py index 70c61625..efec98ac 100644 --- a/structured_prediction_baselines/modules/structured_score/sequence_tagging/cnn.py +++ b/structured_prediction_baselines/modules/structured_score/sequence_tagging/cnn.py @@ -1,4 +1,7 @@ from typing import List, Tuple, Union, Dict, Any, Optional + +from allennlp.nn import Activation + from structured_prediction_baselines.modules.structured_score.structured_score import StructuredScore import torch from structured_prediction_baselines.modules.cnn_encoder import Cnn2dEncoder @@ -6,13 +9,25 @@ @StructuredScore.register("cnn") class CNN(StructuredScore): - def __init__(self, num_tags: int, **kwargs: Any): - """ - TODO: Change kwargs to take hidden size and output size - """ + + def __init__(self, + num_tags: int, + embedding_dim: int, + num_filters: int, + ngram_filter_sizes: Tuple[int, ...], + dropout: float = 0.1, + conv_layer_activation: Activation = None, + output_dim: Optional[int] = None, + **kwargs: Any): super().__init__() self.num_tags = num_tags - self.encoder = Cnn2dEncoder(num_tags, embedding_dim=1, num_filters=50, ngram_filter_sizes=(3,), dropout=0.1) + self.encoder = Cnn2dEncoder(num_tags=num_tags, + embedding_dim=embedding_dim, + num_filters=num_filters, + ngram_filter_sizes=ngram_filter_sizes, + dropout=dropout, + conv_layer_activation=conv_layer_activation, + output_dim=output_dim) def forward( self, @@ -20,7 +35,7 @@ def forward( buffer: Dict, **kwargs: Any, ) -> torch.Tensor: - mask = buffer["mask"] + mask = buffer["mask"] # (batch, seq_len) output = self.encoder(y, mask) # (batch_size, num_samples or 1, ...) output = output.sum(dim=-1) # (batch_size, num_samples or 1, seq_length) output = output * mask.unsqueeze(1) diff --git a/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py b/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py index 6ce11838..577caa7d 100644 --- a/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py +++ b/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py @@ -11,6 +11,7 @@ class SelfAttention(StructuredScore): def __init__(self, num_tags: int, + input_dim: int = None, reduction: str = "max", M: int = 0, num_heads: int = 1, @@ -25,11 +26,12 @@ def __init__(self, self.reduction = reduction self.M = M assert self.M >= 0 - self.attention_dim = attention_dim or num_tags + self.input_dim = input_dim or num_tags + self.attention_dim = attention_dim or self.input_dim self.values_dim = values_dim or self.attention_dim self.attention_layer = SelfAttentionEncoder( num_heads, - input_dim=num_tags, + input_dim=self.input_dim, attention_dim=self.attention_dim, values_dim=self.values_dim, output_projection_dim=output_dim, @@ -86,8 +88,12 @@ def _get_attention_mask(self, n_samples, mask: torch.Tensor): @StructuredScore.register("self-attention-full-sequence") class SelfAttentionFullSequence(SelfAttention): - def __init__(self, num_tags: int, reduction: str = "max", **kwargs: Any): - super().__init__(num_tags, reduction, **kwargs) + def __init__(self, num_tags: int, + input_dim: int, + output_dim: int, + reduction: str = "max", + **kwargs: Any): + super().__init__(num_tags=num_tags, input_dim=input_dim, reduction=reduction, output_dim=output_dim, **kwargs) def _get_attention_mask(self, n_samples: int, mask: torch.Tensor): batch_size, seq_length = mask.shape From d0749a9ad185433f242e55c6f2518da97e736f3f Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 29 Dec 2021 15:57:15 -0500 Subject: [PATCH 06/18] merge conflict --- .../srl/ontonotes_seal_nce_bert.jsonnet | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 model_configs/srl/ontonotes_seal_nce_bert.jsonnet diff --git a/model_configs/srl/ontonotes_seal_nce_bert.jsonnet b/model_configs/srl/ontonotes_seal_nce_bert.jsonnet new file mode 100644 index 00000000..b96c7f53 --- /dev/null +++ b/model_configs/srl/ontonotes_seal_nce_bert.jsonnet @@ -0,0 +1,173 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local ff_weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 25, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + should_log_parameter_statistics: false, + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} From aaa9f114193155f01db36e89099dfe2f1a96369a Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 29 Dec 2021 16:51:42 -0500 Subject: [PATCH 07/18] adds config for ontonotes seal with residual x --- .../ontonotes_seal_nce_bert_residual.jsonnet | 175 ++++++++++++++++++ .../sequence_tagging/self_attention.py | 4 +- 2 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet diff --git a/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet b/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet new file mode 100644 index 00000000..334ec48b --- /dev/null +++ b/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet @@ -0,0 +1,175 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local ff_weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + residual_x: true, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + input_dim: num_labels + transformer_hidden_dim, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 25, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + should_log_parameter_statistics: false, + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py b/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py index 577caa7d..7f883229 100644 --- a/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py +++ b/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py @@ -89,11 +89,9 @@ def _get_attention_mask(self, n_samples, mask: torch.Tensor): @StructuredScore.register("self-attention-full-sequence") class SelfAttentionFullSequence(SelfAttention): def __init__(self, num_tags: int, - input_dim: int, - output_dim: int, reduction: str = "max", **kwargs: Any): - super().__init__(num_tags=num_tags, input_dim=input_dim, reduction=reduction, output_dim=output_dim, **kwargs) + super().__init__(num_tags=num_tags, reduction=reduction, **kwargs) def _get_attention_mask(self, n_samples: int, mask: torch.Tensor): batch_size, seq_length = mask.shape From 164c12954c36a249938a51504e56f3e5981cae1f Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Fri, 31 Dec 2021 11:18:53 -0500 Subject: [PATCH 08/18] adds grad accumulation steps --- model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet b/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet index 334ec48b..c31a3a2c 100644 --- a/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet +++ b/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet @@ -102,7 +102,7 @@ local task_nn = { type: 'seqtag-nce-ranking-with-discrete-sampling', reduction: 'mean', log_key: 'seq_nce_loss', - num_samples: 10, + num_samples: 100, }, initializer: { regexes: [ @@ -115,7 +115,7 @@ local task_nn = { data_loader: { batch_sampler: { type: 'bucket', - batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps sorting_keys: ['tokens'], }, //max_instances_in_memory: if test == '1' then 10 else 1000, @@ -123,6 +123,7 @@ local task_nn = { trainer: { type: 'gradient_descent_minimax', num_epochs: if test == '1' then 10 else 25, + num_gradient_accumulation_steps: 2, grad_norm: { task_nn: 1.0 }, patience: 4, validation_metric: '+f1-measure-overall', From 7a21d42e088e04cfdfcf798ecd238fb11a662965 Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Thu, 6 Jan 2022 01:21:31 -0500 Subject: [PATCH 09/18] variable name chanhges --- .../conll2003_seal_nce_self_attention_bilstm.jsonnet | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet index 58ac84d9..6744681a 100644 --- a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet @@ -18,12 +18,15 @@ local ff_activation = 'softplus'; //local ff_activation = 'softplus'; //local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); -local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); local attention_dim = std.parseJson(std.extVar('attention_dim')); local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local scorenn_lr = std.parseJson(std.extVar('scorenn_lr')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); + local task_nn = { type: 'sequence-tagging', text_field_embedder: { @@ -113,7 +116,7 @@ local task_nn = { normalize_y: false, }, ], - loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], reduction: 'mean', }, }, @@ -169,12 +172,12 @@ local task_nn = { optimizers: { task_nn: { - lr: 0.00001, + lr: tasknn_lr, weight_decay: ff_weight_decay, type: 'adamw', }, score_nn: { - lr: 0.00001, + lr: scorenn_lr, weight_decay: ff_weight_decay, type: 'adamw', }, From e3947de69def4b6d64fcfe6a752d616bc646f088 Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Sun, 9 Jan 2022 17:27:07 -0500 Subject: [PATCH 10/18] Adds multi-task learning feature with shared tasknn (v1) --- ...e_self_attention_bilstm_multi_task.jsonnet | 217 ++++++++++++++++++ .../models/base.py | 103 ++++++++- .../models/multilabel_classification.py | 6 +- .../models/sequence_tagging.py | 12 +- .../modules/sampler/inference_net.py | 16 +- .../modules/score_nn.py | 4 + .../modules/sequence_tagging_score_nn.py | 12 +- .../modules/sequence_tagging_task_nn.py | 28 ++- 8 files changed, 374 insertions(+), 24 deletions(-) create mode 100644 model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet new file mode 100644 index 00000000..2d9aaecb --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet @@ -0,0 +1,217 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_weight_decay = std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local scorenn_lr = std.parseJson(std.extVar('scorenn_lr')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); + +local task_nn = { + type: 'sequence-tagging', + project_onto_tags: false, + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'seal-ner-multi-task', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + tag_projection_layer: { + input_dim: 400, + num_layers: 1, + activations: 'linear', + hidden_dims: num_labels + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + tag_projection_layer: { + input_dim: 400, + num_layers: 1, + activations: 'linear', + hidden_dims: num_labels + }, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: scorenn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/structured_prediction_baselines/models/base.py b/structured_prediction_baselines/models/base.py index 1c19aa39..025286b5 100644 --- a/structured_prediction_baselines/models/base.py +++ b/structured_prediction_baselines/models/base.py @@ -42,10 +42,13 @@ logger = logging.getLogger(__name__) - +@Model.register( + "multi-task-score-based-learning-with-infnet", + constructor="from_partial_objects_with_shared_tasknn" +) @Model.register( "score-based-learning-with-infnet", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @Model.register("score-based-learning", constructor="from_partial_objects") class ScoreBasedLearningModel(LoggingMixin, Model): @@ -208,7 +211,7 @@ def parameters_for_model_mode( yield param @classmethod - def from_partial_objects_with_shared_tasknn( + def from_partial_objects_with_inference_module_as_sampler( cls, vocab: Vocabulary, loss_fn: Lazy[Loss], @@ -224,7 +227,8 @@ def from_partial_objects_with_shared_tasknn( ) -> "ScoreBasedLearningModel": """ This constructor is used only when the `sampler` is an instance of `SamplerContainer` - and we wish to use tasknn as both the inference_module and a sampler in the constituent sampler. + and we wish to use inference_module as a sampler in the constituent sampler i.e. share tasknn with both + infrence_module and sampler """ infnet_sampler = inference_module.construct( inference_nn=task_nn, @@ -296,6 +300,97 @@ def from_partial_objects_with_shared_tasknn( **kwargs, ) + @classmethod + def from_partial_objects_with_shared_tasknn( + cls, + vocab: Vocabulary, + loss_fn: Lazy[Loss], + inference_module: Lazy[Sampler], + task_nn: TaskNN, + sampler: Optional[Lazy[SamplerContainer]] = None, + score_nn: Optional[Lazy[ScoreNN]] = None, + oracle_value_function: Optional[OracleValueFunction] = None, + evaluation_module: Optional[Lazy[Sampler]] = None, + regularizer: Optional[RegularizerApplicator] = None, + initializer: Optional[InitializerApplicator] = None, + **kwargs: Any, + ) -> "ScoreBasedLearningModel": + """ + This constructor is used only when the `sampler` is an instance of `SamplerContainer` + and we wish to use inference_module as a sampler in the constituent sampler and share the tasknn among + the inference_module, sampler and scorenn thereby training the tasknn using `Multi-Task` objective. + """ + score_nn_ = score_nn.construct(task_nn=task_nn) + infnet_sampler = inference_module.construct( + inference_nn=task_nn, + score_nn=score_nn_, + oracle_value_function=oracle_value_function, + ) + + if oracle_value_function is not None: + if sampler is None: + sampler_ = AppendingSamplerContainer( + score_nn=score_nn_, + oracle_value_function=oracle_value_function, + constituent_samplers=[], + log_key="sampler", + ) + else: + sampler_ = sampler.construct( + score_nn=score_nn_, + oracle_value_function=oracle_value_function, + ) + loss_fn_ = loss_fn.construct( + score_nn=score_nn_, oracle_value_function=oracle_value_function + ) + else: + if sampler is None: + sampler_ = AppendingSamplerContainer( + score_nn=score_nn_, + constituent_samplers=[], + log_key="sampler", + ) + else: + + sampler_ = sampler.construct( + score_nn=score_nn_, + ) + loss_fn_ = loss_fn.construct( + score_nn=score_nn_, + ) + # add the infnet sampler + sampler_.append_sampler(infnet_sampler) + + # test-time inference. + # reconstruct the infnet sampler with shared tasknn weights + # to set it as the inference_module + inference_module_ = inference_module.construct( + inference_nn=task_nn, + score_nn=score_nn_, + oracle_value_function=oracle_value_function, + ) + inference_module_.log_key = inference_module_.log_key + "_inf" + + if evaluation_module is not None: + evaluation_module_ = evaluation_module.construct( + score_nn=score_nn_, oracle_value_function=oracle_value_function + ) + else: + evaluation_module_ = None + + return cls( + vocab=vocab, + sampler=sampler_, + loss_fn=loss_fn_, + oracle_value_function=oracle_value_function, + score_nn=score_nn_, + inference_module=inference_module_, + evaluation_module=evaluation_module_, + regularizer=regularizer, + initializer=initializer, + **kwargs, + ) + def calculate_metrics( self, x: Any, diff --git a/structured_prediction_baselines/models/multilabel_classification.py b/structured_prediction_baselines/models/multilabel_classification.py index 74e1c75b..7858d2af 100644 --- a/structured_prediction_baselines/models/multilabel_classification.py +++ b/structured_prediction_baselines/models/multilabel_classification.py @@ -25,14 +25,14 @@ @Model.register( "multi-label-classification-with-infnet", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @Model.register( "multi-label-classification", constructor="from_partial_objects" ) @ScoreBasedLearningModel.register( "multi-label-classification-with-infnet", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @ScoreBasedLearningModel.register( "multi-label-classification", constructor="from_partial_objects" @@ -104,7 +104,7 @@ def get_true_metrics(self, reset: bool = False) -> Dict[str, float]: ) @Model.register( "multi-label-classification-with-infnet-and-scorenn-evaluation", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) class MultilabelClassificationWithScoreNNEvaluation(MultilabelClassification): def __init__( diff --git a/structured_prediction_baselines/models/sequence_tagging.py b/structured_prediction_baselines/models/sequence_tagging.py index 46f11836..be7d2aba 100644 --- a/structured_prediction_baselines/models/sequence_tagging.py +++ b/structured_prediction_baselines/models/sequence_tagging.py @@ -282,9 +282,13 @@ def get_end_transitions(self) -> torch.Tensor: return end_transitions +@Model.register( + "seal-ner-multi-task", + constructor="from_partial_objects_with_shared_tasknn" +) @Model.register( "seal-ner", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @Model.register( "ner-seperate-inference-and-training-network", @@ -365,9 +369,13 @@ def get_metrics(self, reset: bool = False) -> Dict[str, float]: return metrics +@Model.register( + "seal-srl-multi-task", + constructor="from_partial_objects_with_shared_tasknn" +) @Model.register( "seal-srl", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @Model.register( "srl-seperate-inference-and-training-network", diff --git a/structured_prediction_baselines/modules/sampler/inference_net.py b/structured_prediction_baselines/modules/sampler/inference_net.py index 62bbd3fd..65f9900e 100644 --- a/structured_prediction_baselines/modules/sampler/inference_net.py +++ b/structured_prediction_baselines/modules/sampler/inference_net.py @@ -16,6 +16,7 @@ import numpy as np import torch from allennlp.common.lazy import Lazy +from allennlp.modules import FeedForward from allennlp.training.optimizers import Optimizer from structured_prediction_baselines.common import ModelMode from structured_prediction_baselines.modules.loss import Loss @@ -38,6 +39,8 @@ def parameters_with_model_mode( yield from self.inference_nn.parameters() if self.cost_augmented_layer is not None: yield from self.cost_augmented_layer.parameters() + if self.tag_projection_layer is not None: + yield from self.tag_projection_layer.parameters() def __init__( self, @@ -46,6 +49,7 @@ def __init__( score_nn: ScoreNN, cost_augmented_layer: Optional[CostAugmentedLayer] = None, oracle_value_function: Optional[OracleValueFunction] = None, + tag_projection_layer: Optional[FeedForward] = None, **kwargs: Any, ): assert ScoreNN is not None @@ -57,6 +61,7 @@ def __init__( self.inference_nn = inference_nn self.cost_augmented_layer = cost_augmented_layer self.loss_fn = loss_fn + self.tag_projection_layer = tag_projection_layer self.logging_children.append(self.loss_fn) @@ -113,9 +118,14 @@ def _get_values( **kwargs: Any, ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - y_inf: torch.Tensor = self.inference_nn(x, buffer).unsqueeze( - 1 - ) # (batch_size, 1, ...) unormalized + if self.tag_projection_layer: + y_inf: torch.Tensor = self.tag_projection_layer( + self.inference_nn(x, buffer) + ).unsqueeze(1) # (batch_size, 1, ...) unormalized + else: + y_inf: torch.Tensor = self.inference_nn(x, buffer).unsqueeze( + 1 + ) # (batch_size, 1, ...) unormalized # inference_nn is TaskNN so it will output tensor of shape (batch, ...) # hence the unsqueeze diff --git a/structured_prediction_baselines/modules/score_nn.py b/structured_prediction_baselines/modules/score_nn.py index 615d5de6..835e9dae 100644 --- a/structured_prediction_baselines/modules/score_nn.py +++ b/structured_prediction_baselines/modules/score_nn.py @@ -1,6 +1,8 @@ from typing import List, Tuple, Union, Dict, Any, Optional from allennlp.common.registrable import Registrable import torch +from allennlp.modules import FeedForward + from .task_nn import TaskNN from .structured_score.structured_score import StructuredScore @@ -13,10 +15,12 @@ def __init__( task_nn: TaskNN, # (batch, ...) global_score: Optional[StructuredScore] = None, residual_x: bool = False, + tag_projection_layer: Optional[FeedForward] = None, **kwargs: Any, ): super().__init__() # type:ignore self.task_nn = task_nn + self.tag_projection_layer = tag_projection_layer self.residual_x = residual_x self.global_score = global_score self._dtype = self.compute_input_dtype() diff --git a/structured_prediction_baselines/modules/sequence_tagging_score_nn.py b/structured_prediction_baselines/modules/sequence_tagging_score_nn.py index d12ddb98..4c7fb728 100644 --- a/structured_prediction_baselines/modules/sequence_tagging_score_nn.py +++ b/structured_prediction_baselines/modules/sequence_tagging_score_nn.py @@ -25,10 +25,14 @@ def compute_local_score( # type:ignore # x, buffer # ) # (batch, ...) of unormalized logits # buffer["y_local"] = y_local - - y_local = self.task_nn( - x, buffer - ) # (batch, ...) of unormalized logits + if self.tag_projection_layer: + y_local = self.tag_projection_layer( + self.task_nn(x, buffer) + ) # (batch, ...) of unormalized logits + else: + y_local = self.task_nn( + x, buffer + ) # (batch, ...) of unormalized logits mask = buffer.get("mask") mask = mask.unsqueeze(1) diff --git a/structured_prediction_baselines/modules/sequence_tagging_task_nn.py b/structured_prediction_baselines/modules/sequence_tagging_task_nn.py index 4a5670ef..d7e2b922 100644 --- a/structured_prediction_baselines/modules/sequence_tagging_task_nn.py +++ b/structured_prediction_baselines/modules/sequence_tagging_task_nn.py @@ -1,3 +1,5 @@ +import logging + from .task_nn import TaskNN, CostAugmentedLayer from typing import List, Tuple, Union, Dict, Any, Optional import torch @@ -12,6 +14,8 @@ import torch.nn.functional as F import allennlp.nn.util as util +logger = logging.getLogger(__name__) + @TaskNN.register("sequence-tagging") class SequenceTaggingTaskNN(TaskNN): @@ -23,6 +27,8 @@ def __init__( feedforward: Optional[FeedForward] = None, dropout: float = 0, label_namespace: str = "labels", + project_onto_tags: bool = True + ): """ @@ -50,10 +56,13 @@ def __init__( if output_dim is None: raise ValueError("output_dim cannot be None") - self.tag_projection_layer = TimeDistributed( - Linear(output_dim, self.num_tags) - ) # equivalent to Uj.b(x,t) in eq (3) - + if project_onto_tags: + self.tag_projection_layer = TimeDistributed( + Linear(output_dim, self.num_tags) + ) # equivalent to Uj.b(x,t) in eq (3) + else: + self.tag_projection_layer = None + logger.warning("Tag Projection Layer set to None. TaskNN would return encoded text instead of logits.") if dropout: self.dropout: Optional[torch.nn.Module] = torch.nn.Dropout(dropout) else: @@ -83,11 +92,14 @@ def forward( if self.feedforward: encoded_text = self.feedforward(encoded_text) - logits = self.tag_projection_layer(encoded_text) + if self.tag_projection_layer: + logits = self.tag_projection_layer(encoded_text) + + return ( + logits # shape (batch, sequence, num_tags) of unormalized logits + ) - return ( - logits # shape (batch, sequence, num_tags) of unormalized logits - ) + return encoded_text # shape (batch, sequence, output_dim) @CostAugmentedLayer.register("sequence-tagging-stacked") From 5611b3f66366a2d54cb73d577fff01e870c7cd7e Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Sun, 9 Jan 2022 17:47:39 -0500 Subject: [PATCH 11/18] fix omw test error --- tests/end2end/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/end2end/conftest.py b/tests/end2end/conftest.py index 5811606f..5cf24f5f 100644 --- a/tests/end2end/conftest.py +++ b/tests/end2end/conftest.py @@ -1,4 +1,10 @@ from .common import marks +import nltk + +try: + nltk.data.find('corpora/omw-1.4') +except LookupError: + nltk.download('omw-1.4') def pytest_configure(config) -> None: # type: ignore From 6b2591b59b54e037aed54db94218b42f862617da Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 19 Jan 2022 01:29:08 -0500 Subject: [PATCH 12/18] Adds sweep for ner bilstm multi-task --- ...003_seal_nce_self_attention_bilstm.jsonnet | 2 +- ..._nce_self_attention_bilstm_multi_task.yaml | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet index 6744681a..9519bd9c 100644 --- a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet @@ -7,7 +7,7 @@ local dataset_name = 'conll2003ner'; local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; local num_labels = dataset_metadata.num_labels; // local transformer_model = 'bert-base-uncased'; -// local transformer_hidden_dim = 768; +// local s = 768; // local max_length = 256; //local ff_hidden = std.parseJson(std.extVar('ff_hidden')); diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml new file mode 100644 index 00000000..a6190977 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml @@ -0,0 +1,47 @@ +name: conll2003_seal_nce_discrete_self_attention_bilstm_multi_task +program: allennlp +command: +- ${program} +- train-with-wandb +- model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="mode=multi_task,task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + model.loss_fn.num_samples: + value: 100 + env.attention_dim: + distribution: q_uniform + min: 49 # 50 + max: 249 # 200 + q: 50 + env.attention_dropout_10x: + distribution: q_uniform + min: 0.5 # 1 + max: 5.49 # 5 \ No newline at end of file From 0f6ee543f4cf2d116e75c9374d11390f384f7abf Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 19 Jan 2022 01:29:59 -0500 Subject: [PATCH 13/18] corrects the config file name --- .../conll2003_seal_nce_self_attention_bilstm_multi_task.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml index a6190977..03b7dd98 100644 --- a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml @@ -3,7 +3,7 @@ program: allennlp command: - ${program} - train-with-wandb -- model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet +- model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet - --include-package=structured_prediction_baselines - --wandb_tags="mode=multi_task,task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" - ${args} From b2c0e243268cb89367304d18fd59b5653e26261b Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 19 Jan 2022 01:45:46 -0500 Subject: [PATCH 14/18] Adds ner bilstm residual model config --- ...nce_self_attention_bilstm_residual.jsonnet | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet new file mode 100644 index 00000000..a5b27a9d --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet @@ -0,0 +1,207 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local scorenn_lr = std.parseJson(std.extVar('scorenn_lr')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + residual_x: true, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + input_dim: 50+128+17, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 100, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: scorenn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file From 48807db87876db561ea785649e8a444c8d0221fd Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 19 Jan 2022 01:53:35 -0500 Subject: [PATCH 15/18] changes wrt to latest wandb version --- .../conll2003_seal_nce_self_attention_bilstm_multi_task.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml index 03b7dd98..b84588ad 100644 --- a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml @@ -5,7 +5,7 @@ command: - train-with-wandb - model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet - --include-package=structured_prediction_baselines -- --wandb_tags="mode=multi_task,task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" +- --wandb-tags="mode=multi_task,task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" - ${args} - --file-friendly-logging method: bayes From 15ef24f2f2200b6a45b80626a7d3f1c080a7616c Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 19 Jan 2022 01:59:48 -0500 Subject: [PATCH 16/18] updates env variable names --- ...conll2003_seal_nce_self_attention_bilstm_multi_task.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml index b84588ad..dfca5c99 100644 --- a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml @@ -20,15 +20,17 @@ early_terminate: parameters: env.cross_entropy_loss_weight: value: 1.0 + env.ff_weight_decay: + value: 1e-5 env.score_loss_weight: distribution: log_uniform min: -6.9 max: 2.3 - trainer.optimizer.optimizers.task_nn.lr: + env.task_nn_lr: distribution: log_uniform min: -12.5 max: -4.5 - trainer.optimizer.optimizers.score_nn.lr: + env.score_nn.lr: distribution: log_uniform min: -11.5 max: -4.5 From 75f9ee4f119765abef9a61483a16aa5fc9e4a60f Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Wed, 19 Jan 2022 02:03:39 -0500 Subject: [PATCH 17/18] updates env variable names --- ...2003_seal_nce_self_attention_bilstm_multi_task.jsonnet | 8 ++++---- ...nll2003_seal_nce_self_attention_bilstm_multi_task.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet index 2d9aaecb..21c1f031 100644 --- a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet @@ -24,8 +24,8 @@ local ff_weight_decay = std.parseJson(std.extVar('ff_weight_decay')); local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); -local scorenn_lr = std.parseJson(std.extVar('scorenn_lr')); -local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); +local score_nn_lr = std.parseJson(std.extVar('score_nn_lr')); +local task_nn_lr = std.parseJson(std.extVar('task_nn_lr')); local task_nn = { type: 'sequence-tagging', @@ -184,12 +184,12 @@ local task_nn = { optimizers: { task_nn: { - lr: tasknn_lr, + lr: task_nn_lr, weight_decay: ff_weight_decay, type: 'adamw', }, score_nn: { - lr: scorenn_lr, + lr: score_nn_lr, weight_decay: ff_weight_decay, type: 'adamw', }, diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml index dfca5c99..d2491cef 100644 --- a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml @@ -30,7 +30,7 @@ parameters: distribution: log_uniform min: -12.5 max: -4.5 - env.score_nn.lr: + env.score_nn_lr: distribution: log_uniform min: -11.5 max: -4.5 From 21f97b512d21db37ab5e97c91c9b405d50f59480 Mon Sep 17 00:00:00 2001 From: Purujit Goyal Date: Thu, 20 Jan 2022 00:29:45 -0500 Subject: [PATCH 18/18] Adds sweep config for ner bilstm residual --- ...nce_self_attention_bilstm_residual.jsonnet | 2 +- ...al_nce_self_attention_bilstm_residual.yaml | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.yaml diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet index a5b27a9d..d16ec4d6 100644 --- a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet @@ -20,7 +20,7 @@ local ff_activation = 'softplus'; local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); local attention_dim = std.parseJson(std.extVar('attention_dim')); -local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local ff_weight_decay = std.parseJson(std.extVar('ff_weight_decay')); local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.yaml new file mode 100644 index 00000000..57299f23 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.yaml @@ -0,0 +1,49 @@ +name: conll2003_seal_nce_discrete_self_attention_bilstm_residual +program: allennlp +command: +- ${program} +- train-with-wandb +- model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet +- --include-package=structured_prediction_baselines +- --wandb-tags="mode=residual,task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.ff_weight_decay: + value: 1e-5 + env.score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + env.tasknn_lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + env.scorenn_lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + model.loss_fn.num_samples: + value: 100 + env.attention_dim: + distribution: q_uniform + min: 49 # 50 + max: 249 # 200 + q: 50 + env.attention_dropout_10x: + distribution: q_uniform + min: 0.5 # 1 + max: 5.49 # 5 \ No newline at end of file