diff --git a/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_dvn.jsonnet b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_dvn.jsonnet new file mode 100644 index 00000000..d3d573cc --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_dvn.jsonnet @@ -0,0 +1,240 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'bgc'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [ + //GBI + { + type: 'gradient-based-inference', + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss // change this + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + // Adversarial + { + type: 'gradient-based-inference', + log_key: 'adv', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'negative', + log_key: 'neg', + constituent_loss: { type: 'multi-label-dvn-bce', reduction: 'none', log_key: 'dvn_bce' }, + reduction: 'none', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + + { type: 'ground-truth' }, + ], + }, + inference_module: { + type: 'gradient-based-inference', + log_key: 'inference', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 30, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { type: 'multi-label-dvn-bce', log_key: 'dvn_bce' }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..356f8e87 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet @@ -0,0 +1,241 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'bgc'; +local data_reader_name = (if std.toString(dataset_name) == 'nyt10' then 'nyt' else dataset_name); # nyt10 or bgc +local data_reader_name = (if std.toString(dataset_name) == 'rcv1' then 'rcv' else data_reader_name); # nyt10 or rcv1 + +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local inference_score_weight = std.parseJson(std.extVar('inference_score_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-inference-net-normalized', + log_key: 'inference_module', + cost_augmented_layer: { + type: 'multi-label-stacked', + feedforward: { + input_dim: 2 * num_labels, + num_layers: 2, + activations: [ff_activation, 'linear'], + hidden_dims: num_labels, + }, + normalize_y: true, + }, + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-inference', + log_key: 'neg_inference', + normalize_y: true, + reduction: 'none', + inference_score_weight: inference_score_weight, + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [1.0, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { + type: 'manhattan', + differentiable: true, + }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-margin-based', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + num_gradient_accumulation_steps: 2, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet index a4e7f8c1..995963dd 100644 --- a/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet +++ b/model_configs/multilabel_classification/v2.5/bgc_bert_adapter_nce_bert_adapter_tasknn.jsonnet @@ -30,6 +30,9 @@ local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_w local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); + + + local feature_network = { text_field_embedder: { token_embedders: { @@ -71,7 +74,6 @@ local feature_network = { model_name: transformer_model, max_length: 512, }, - }, train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + dataset_metadata.train_file), @@ -79,7 +81,6 @@ local feature_network = { dataset_metadata.validation_file), test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + dataset_metadata.test_file), - vocabulary: { type: 'from_files', directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' diff --git a/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_dvn.jsonnet b/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_dvn.jsonnet new file mode 100644 index 00000000..a86749d2 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_dvn.jsonnet @@ -0,0 +1,236 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [ + //GBI + { + type: 'gradient-based-inference', + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss // change this + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + // Adversarial + { + type: 'gradient-based-inference', + log_key: 'adv', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'negative', + log_key: 'neg', + constituent_loss: { type: 'multi-label-dvn-bce', reduction: 'none', log_key: 'dvn_bce' }, + reduction: 'none', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + + { type: 'ground-truth' }, + ], + }, + inference_module: { + type: 'gradient-based-inference', + log_key: 'inference', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 30, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { type: 'multi-label-dvn-bce', log_key: 'dvn_bce' }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + shuffle: true, + batch_size: 32, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..61396ee7 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/gendata_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet @@ -0,0 +1,242 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; +// local dataset_name = std.parseJson(std.extVar('dataset_name')); +local data_reader_name = (if std.toString(dataset_name) == 'nyt10' then 'nyt' else dataset_name); // nyt10 or bgc +local data_reader_name = (if std.toString(dataset_name) == 'rcv1' then 'rcv' else data_reader_name); // nyt10 or rcv1 + +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local inference_score_weight = std.parseJson(std.extVar('inference_score_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'bgc', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-inference-net-normalized', + log_key: 'inference_module', + cost_augmented_layer: { + type: 'multi-label-stacked', + feedforward: { + input_dim: 2 * num_labels, + num_layers: 2, + activations: [ff_activation, 'linear'], + hidden_dims: num_labels, + }, + normalize_y: true, + }, + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-inference', + log_key: 'neg_inference', + normalize_y: true, + reduction: 'none', + inference_score_weight: inference_score_weight, + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [1.0, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { + type: 'manhattan', + differentiable: true, + }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-margin-based', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + //num_gradient_accumulation_steps: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_dvn.jsonnet b/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_dvn.jsonnet new file mode 100644 index 00000000..2b2b8621 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_dvn.jsonnet @@ -0,0 +1,240 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'nyt10'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'nyt', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [ + //GBI + { + type: 'gradient-based-inference', + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss // change this + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + // Adversarial + { + type: 'gradient-based-inference', + log_key: 'adv', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'negative', + log_key: 'neg', + constituent_loss: { type: 'multi-label-dvn-bce', reduction: 'none', log_key: 'dvn_bce' }, + reduction: 'none', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + + { type: 'ground-truth' }, + ], + }, + inference_module: { + type: 'gradient-based-inference', + log_key: 'inference', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 30, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { type: 'multi-label-dvn-bce', log_key: 'dvn_bce' }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..9bd4434f --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/nyt_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet @@ -0,0 +1,238 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'nyt10'; +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local inference_score_weight = std.parseJson(std.extVar('inference_score_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'nyt', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-inference-net-normalized', + log_key: 'inference_module', + cost_augmented_layer: { + type: 'multi-label-stacked', + feedforward: { + input_dim: 2 * num_labels, + num_layers: 2, + activations: [ff_activation, 'linear'], + hidden_dims: num_labels, + }, + normalize_y: true, + }, + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-inference', + log_key: 'neg_inference', + normalize_y: true, + reduction: 'none', + inference_score_weight: inference_score_weight, + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [1.0, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { + type: 'manhattan', + differentiable: true, + }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-margin-based', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + //num_gradient_accumulation_steps: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_dvn.jsonnet b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_dvn.jsonnet new file mode 100644 index 00000000..9ee7b2a1 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_dvn.jsonnet @@ -0,0 +1,240 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'rcv', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [ + //GBI + { + type: 'gradient-based-inference', + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss // change this + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + // Adversarial + { + type: 'gradient-based-inference', + log_key: 'adv', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'negative', + log_key: 'neg', + constituent_loss: { type: 'multi-label-dvn-bce', reduction: 'none', log_key: 'dvn_bce' }, + reduction: 'none', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + + { type: 'ground-truth' }, + ], + }, + inference_module: { + type: 'gradient-based-inference', + log_key: 'inference', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { type: 'multi-label-dvn-score', reduction: 'none', log_key: 'neg_dvn_score' }, //This loss can be different from the main loss + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 30, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { type: 'multi-label-dvn-bce', log_key: 'dvn_bce' }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..27810b78 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_margin_l1_bert_adapter_tasknn.jsonnet @@ -0,0 +1,238 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local inference_score_weight = std.parseJson(std.extVar('inference_score_weight')); + + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'rcv', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification-with-infnet', + + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + inference_module: { + type: 'multi-label-inference-net-normalized', + log_key: 'inference_module', + cost_augmented_layer: { + type: 'multi-label-stacked', + feedforward: { + input_dim: 2 * num_labels, + num_layers: 2, + activations: [ff_activation, 'linear'], + hidden_dims: num_labels, + }, + normalize_y: true, + }, + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'multi-label-inference', + log_key: 'neg_inference', + normalize_y: true, + reduction: 'none', + inference_score_weight: inference_score_weight, + }, //This loss can be different from the main loss // change this + { + type: 'multi-label-bce', + reduction: 'none', + log_key: 'bce', + }, + ], + loss_weights: [1.0, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { + type: 'manhattan', + differentiable: true, + }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-margin-based', + oracle_cost_weight: 1.0, + perceptron_loss_weight: inference_score_weight, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*feedforward._linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 1 else 300, + grad_norm: { task_nn: 1.0, score_nn: 1.0 }, + num_gradient_accumulation_steps: 2, // effective batch size = batch_size*num_gradient_accumulation_steps + patience: 4, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 1, + verbose: true, + }, + }, + optimizer: { + optimizers: { // have only tasknn optmizer + task_nn: { + lr: 1e-5, + weight_decay: task_nn_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_spen.jsonnet b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_spen.jsonnet new file mode 100644 index 00000000..4fa86801 --- /dev/null +++ b/model_configs/multilabel_classification/v2.5/rcv_bert_adapter_spen.jsonnet @@ -0,0 +1,247 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'rcv1'; //std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import '../datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local num_input_features = dataset_metadata.input_features; + +// model variables +// // common +local ff_activation = 'softplus'; +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local ff_linear_layers = 2; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local dropout = std.parseJson(std.extVar('dropout_10x')) / 10.0; + +// // score_nn +local transformer_model = 'bert-base-uncased'; // huggingface name of the model +local transformer_dim = 768; +local transformer_vocab_size = 30522; +local score_nn_weight_decay = weight_decay; +local global_score_hidden_dim = std.parseJson(std.extVar('global_score_hidden_dim')); +local score_nn_dropout = dropout; +// // task_nn +local task_nn_dropout = dropout; +local task_nn_weight_decay = weight_decay; + +// model variables +local gbi_lr = std.parseJson(std.extVar('gbi_lr')); +local gbi_optim = 'adam'; + +local feature_network = { + text_field_embedder: { + token_embedders: { + x: { + type: 'pretrained_transformer_with_adapter', + model_name: transformer_model, + }, + }, + }, + seq2vec_encoder: { + type: 'bert_pooler', + pretrained_model: transformer_model, + }, + final_dropout: 0, + feedforward: { + input_dim: transformer_dim, + num_layers: ff_linear_layers, + activations: ([ff_activation for i in std.range(0, ff_linear_layers - 2)] + [ff_activation]), + hidden_dims: ([transformer_dim * 2 for i in std.range(0, ff_linear_layers - 2)] + [transformer_dim]), + dropout: ([task_nn_dropout for i in std.range(0, ff_linear_layers - 2)] + [0]), + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + // Data + dataset_reader: { + type: 'rcv', + //[if test == '1' then 'max_instances']: 100, + token_indexers: { + x: { + type: 'pretrained_transformer', + model_name: transformer_model, + }, + }, + tokenizer: { + type: 'pretrained_transformer', + model_name: transformer_model, + max_length: 512, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + +vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab' + }, + // Model + model: { + type: 'multi-label-classification', + sampler: { + type: 'gradient-based-inference-tasknn-init', + log_key: 'sampler', + inference_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + gbi_sampler: //GBI + { + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'multi-label-inference', + oracle_cost_weight: 1.0, + inference_score_weight: 1, + log_key: 'margin_loss', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + }, + inference_module: { + type: 'gradient-based-inference-tasknn-init', + log_key: 'sampler', + inference_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + gbi_sampler: //GBI + { + log_key: 'gbi', + gradient_descent_loop: { + optimizer: { + lr: gbi_lr, //0.1 + weight_decay: 0, + type: gbi_optim, + }, + }, + loss_fn: { + type: 'multi-label-score-loss', + reduction: 'none', + log_key: 'score_loss', + }, + output_space: { type: 'multi-label-relaxed', num_labels: num_labels, default_value: 0.0 }, + stopping_criteria: 20, + sample_picker: { type: 'best' }, + number_init_samples: 1, + random_mixing_in_init: 1.0, + }, + }, + oracle_value_function: { type: 'per-instance-f1', differentiable: false }, + score_nn: { + type: 'multi-label-classification', + task_nn: { + type: 'multi-label-text-classification', + feature_network: feature_network, + label_embeddings: { + embedding_dim: transformer_dim, + vocab_namespace: 'labels', + }, + }, + global_score: { + type: 'multi-label-feedforward', + feedforward: { + input_dim: num_labels, + num_layers: 1, + activations: ff_activation, + hidden_dims: global_score_hidden_dim, + }, + }, + }, + loss_fn: { + type: 'multi-label-structured-svm', + oracle_cost_weight: 1.0, + perceptron_loss_weight: 1, + reduction: 'mean', + log_key: 'margin_loss', + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*_linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 8, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['x'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { score_nn: 10.0 }, + num_gradient_accumulation_steps: 2, + patience: 20, + validation_metric: '+fixed_f1', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + score_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + score_nn: { + lr: 5e-5, + weight_decay: score_nn_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + }, + ] + else [] + ), + inner_mode: 'task_nn', + num_steps: { task_nn: 0, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet new file mode 100644 index 00000000..dbcdb582 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_cnn.jsonnet @@ -0,0 +1,172 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +local max_length = 512; + +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_mismatched_with_adapter', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'pretrained_transformer_mismatched', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'cnn', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 200, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet new file mode 100644 index 00000000..581921d4 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_seal_nce_linearchain.jsonnet @@ -0,0 +1,171 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +local max_length = 512; + +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_mismatched_with_adapter', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'pretrained_transformer_mismatched', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 200, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/conll2003_bert_adapter_tasknn.jsonnet b/model_configs/sequence_tagging/conll2003_bert_adapter_tasknn.jsonnet new file mode 100644 index 00000000..aa82a443 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_bert_adapter_tasknn.jsonnet @@ -0,0 +1,137 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +local max_length = 512; + +local ff_activation = 'softplus'; +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_mismatched_with_adapter', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'pretrained_transformer_mismatched', + model_name: transformer_model, + max_length: max_length, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'mean', // mean will work fine because seq-tagging-masked-ce will take care of masking + normalize_y: false, // don't normalize because ce requires logits + }, + }, + loss_fn: { + type: 'zero', // there is no score_nn so we put a dummy zero loss + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + num_workers: 5, + max_instances_in_memory: if test == '1' then 10 else 1000, + start_method: 'spawn', + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet b/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet index f7c82850..ed694616 100644 --- a/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet +++ b/model_configs/sequence_tagging/conll2003_inference_net_stacked.jsonnet @@ -63,7 +63,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { - type: 'sequence-tagging-with-infnet', + type: 'seal-ner', label_encoding: 'BIOUL', sampler: { type: 'appending-container', @@ -114,9 +114,11 @@ local task_nn = { score_nn: { type: 'sequence-tagging', task_nn: task_nn, + residual_x: true, global_score: { type: 'self-attention-full-sequence', num_tags: num_labels, + input_dim: num_labels + transformer_hidden_dim, num_heads: 1, attention_dim: 300, output_dim: 100, diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet new file mode 100644 index 00000000..3dd703e8 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet @@ -0,0 +1,198 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +//local ff_dropout = std.parseJson(std.extVar('ff_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'cnn', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 200, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet new file mode 100644 index 00000000..c14f0d34 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_linearchain_bilstm.jsonnet @@ -0,0 +1,198 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +//local ff_dropout = std.parseJson(std.extVar('ff_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet new file mode 100644 index 00000000..9519bd9c --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet @@ -0,0 +1,205 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local s = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local scorenn_lr = std.parseJson(std.extVar('scorenn_lr')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: scorenn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet new file mode 100644 index 00000000..21c1f031 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet @@ -0,0 +1,217 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_weight_decay = std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local score_nn_lr = std.parseJson(std.extVar('score_nn_lr')); +local task_nn_lr = std.parseJson(std.extVar('task_nn_lr')); + +local task_nn = { + type: 'sequence-tagging', + project_onto_tags: false, + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'seal-ner-multi-task', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + tag_projection_layer: { + input_dim: 400, + num_layers: 1, + activations: 'linear', + hidden_dims: num_labels + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + tag_projection_layer: { + input_dim: 400, + num_layers: 1, + activations: 'linear', + hidden_dims: num_labels + }, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: task_nn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: score_nn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet new file mode 100644 index 00000000..d16ec4d6 --- /dev/null +++ b/model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet @@ -0,0 +1,207 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'conll2003ner'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_weight_decay = std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local scorenn_lr = std.parseJson(std.extVar('scorenn_lr')); +local tasknn_lr = std.parseJson(std.extVar('tasknn_lr')); + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'conll2003', + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'seal-ner', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + residual_x: true, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + input_dim: 50+128+17, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 100, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: tasknn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: scorenn_lr, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet b/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet index b1f508e9..6aa2fac1 100644 --- a/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet +++ b/model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet @@ -6,9 +6,7 @@ local use_wandb = (if test == '1' then false else true); local dataset_name = 'conll2003ner'; local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; local num_labels = dataset_metadata.num_labels; -//local transformer_model = 'bert-base-uncased'; -//local transformer_hidden_dim = 768; -//local max_length = 256; + //local ff_hidden = std.parseJson(std.extVar('ff_hidden')); //local label_space_dim = ff_hidden; @@ -83,7 +81,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { - type: 'sequence-tagging-with-infnet', + type: 'seal-ner', label_encoding: 'BIOUL', sampler: { type: 'appending-container', @@ -184,4 +182,8 @@ local task_nn = { inner_mode: 'score_nn', num_steps: { task_nn: 1, score_nn: 1 }, }, +<<<<<<< HEAD +} +======= } +>>>>>>> fd90a955d9edae124992ebbba14c219abe92180b diff --git a/model_configs/sequence_tagging/conll2003_tasknn_ff.jsonnet b/model_configs/sequence_tagging/conll2003_tasknn_ff.jsonnet index 59e7ccd5..46c4e6b2 100644 --- a/model_configs/sequence_tagging/conll2003_tasknn_ff.jsonnet +++ b/model_configs/sequence_tagging/conll2003_tasknn_ff.jsonnet @@ -65,7 +65,7 @@ local task_nn = { dataset_metadata.test_file), // Model model: { - type: 'sequence-tagging-with-infnet', + type: 'seal-ner', label_encoding: 'BIOUL', sampler: { type: 'appending-container', diff --git a/model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet b/model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet new file mode 100644 index 00000000..9d4b88ae --- /dev/null +++ b/model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet @@ -0,0 +1,198 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = std.parseJson(std.extVar('dataset_name')); +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = dataset_metadata.num_labels; +// local transformer_model = 'bert-base-uncased'; +// local transformer_hidden_dim = 768; +// local max_length = 256; + +//local ff_hidden = std.parseJson(std.extVar('ff_hidden')); +//local label_space_dim = ff_hidden; +//local ff_dropout = std.parseJson(std.extVar('ff_dropout_10x'))/10.0; +//local ff_activation = std.parseJson(std.extVar('ff_activation')); +local ff_activation = 'softplus'; +//local ff_activation = 'softplus'; +//local ff_linear_layers = std.parseJson(std.extVar('ff_linear_layers')); +local cross_entropy_loss_weight = std.parseJson(std.extVar('cross_entropy_loss_weight')); +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local ff_weight_decay = 0.0001; //std.parseJson(std.extVar('ff_weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'embedding', + embedding_dim: 50, + pretrained_file: 'https://allennlp.s3.amazonaws.com/datasets/glove/glove.6B.50d.txt.gz', + trainable: true, + }, + token_characters: { + type: 'character_encoding', + embedding: { + embedding_dim: 16, + }, + encoder: { + type: 'cnn', + embedding_dim: 16, + num_filters: 128, + ngram_filter_sizes: [3], + conv_layer_activation: 'relu', + }, + }, + }, + }, + encoder: { + type: 'lstm', + input_size: 50+128, + hidden_size: 200, + num_layers: 2, + dropout: 0.5, + bidirectional: true, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: dataset_name, + tag_label: 'ner', + coding_scheme: 'BIOUL', + token_indexers: { + tokens: { + type: 'single_id', + lowercase_tokens: true, + }, + token_characters: { + type: 'characters', + min_padding_length: 3, + }, + }, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + // Model + model: { + type: 'sequence-tagging-with-infnet', + label_encoding: 'BIOUL', + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', // bucket is only good for tasks that involve seq + batch_size: 32, + }, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 300, + grad_norm: { task_nn: 1.0 }, + patience: 20, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 5, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: score_nn_steps }, + }, +} \ No newline at end of file diff --git a/model_configs/srl/ontonotes_bert_adapter_seal_nce_cnn.jsonnet b/model_configs/srl/ontonotes_bert_adapter_seal_nce_cnn.jsonnet new file mode 100644 index 00000000..23cee3ff --- /dev/null +++ b/model_configs/srl/ontonotes_bert_adapter_seal_nce_cnn.jsonnet @@ -0,0 +1,166 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); + + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_with_adapter', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'cnn', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 200, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: weight_decay, + type: 'adamw', + }, + + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/srl/ontonotes_bert_adapter_seal_nce_linearchain.jsonnet b/model_configs/srl/ontonotes_bert_adapter_seal_nce_linearchain.jsonnet new file mode 100644 index 00000000..c6f573f5 --- /dev/null +++ b/model_configs/srl/ontonotes_bert_adapter_seal_nce_linearchain.jsonnet @@ -0,0 +1,166 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local dvn_score_loss_weight = std.parseJson(std.extVar('dvn_score_loss_weight')); +local weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local score_temp = std.parseJson(std.extVar('score_nn_steps')); # variable for score_nn.steps +local score_nn_steps = (if std.toString(score_temp) == '0' then 1 else score_temp); + + +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer_with_adapter', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [dvn_score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'linear-chain', //type: 'linear-chain', + num_tags: num_labels, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 200, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 50, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: weight_decay, + type: 'adamw', + }, + + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/srl/ontonotes_seal_nce_bert.jsonnet b/model_configs/srl/ontonotes_seal_nce_bert.jsonnet new file mode 100644 index 00000000..b96c7f53 --- /dev/null +++ b/model_configs/srl/ontonotes_seal_nce_bert.jsonnet @@ -0,0 +1,173 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local ff_weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 10, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 32, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 25, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + should_log_parameter_statistics: false, + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet b/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet new file mode 100644 index 00000000..c31a3a2c --- /dev/null +++ b/model_configs/srl/ontonotes_seal_nce_bert_residual.jsonnet @@ -0,0 +1,176 @@ +local test = std.extVar('TEST'); // a test run with small dataset +local data_dir = std.extVar('DATA_DIR'); +local cuda_device = std.extVar('CUDA_DEVICE'); +local use_wandb = (if test == '1' then false else true); + +local dataset_name = 'ontonotes'; +local dataset_metadata = (import 'datasets.jsonnet')[dataset_name]; +local num_labels = 130; +local transformer_model = 'bert-base-uncased'; +local transformer_hidden_dim = 768; +//local max_length = 512; + +local attention_dropout = std.parseJson(std.extVar('attention_dropout_10x'))/10.0; +local attention_dim = std.parseJson(std.extVar('attention_dim')); +local ff_activation = 'softplus'; +local cross_entropy_loss_weight = 1; +local score_loss_weight = std.parseJson(std.extVar('score_loss_weight')); +local ff_weight_decay = std.parseJson(std.extVar('weight_decay')); +local gain = (if ff_activation == 'tanh' then 5 / 3 else 1); +local task_nn = { + type: 'sequence-tagging', + text_field_embedder: { + token_embedders: { + tokens: { + type: 'pretrained_transformer', // we don't use mismatched because that is what allennlp srl model does + model_name: transformer_model, + //max_length: max_length, + }, + }, + }, +}; + +{ + [if use_wandb then 'type']: 'train_test_log_to_wandb', + evaluate_on_test: true, + dataset_reader: { + type: 'srl', + bert_model_name: transformer_model, + //[if test == '1' then 'max_instances']: 100, + }, + train_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.train_file), + validation_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.validation_file), + test_data_path: (data_dir + '/' + dataset_metadata.dir_name + '/' + + dataset_metadata.test_file), + vocabulary: { + type: 'from_files', + directory: data_dir + '/' + dataset_metadata.dir_name + '/' + 'bert_vocab', + }, + // Model + model: { + type: 'seal-srl', + label_encoding: 'BIO', + using_bert_encoder: true, + decode_on_wordpieces: true, + sampler: { + type: 'appending-container', + log_key: 'sampler', + constituent_samplers: [], + }, + task_nn: task_nn, + inference_module: { + type: 'sequence-tagging-inference-net-normalized', + log_key: 'inference_module', + loss_fn: { + type: 'combination-loss', + log_key: 'loss', + constituent_losses: [ + { + type: 'sequence-tagging-score-loss', // jy: fix here.. + log_key: 'neg.nce_score', + normalize_y: true, + reduction: 'none', + }, //This loss can be different from the main loss // change this + { + type: 'sequence-tagging-masked-cross-entropy', + log_key: 'ce', + reduction: 'none', + normalize_y: false, + }, + ], + loss_weights: [score_loss_weight, cross_entropy_loss_weight], + reduction: 'mean', + }, + }, + oracle_value_function: { type: 'manhattan', differentiable: true}, + score_nn: { + type: 'sequence-tagging', + task_nn: task_nn, + residual_x: true, + global_score: { + type: 'self-attention-full-sequence', + num_heads: 1, + num_tags: num_labels, + input_dim: num_labels + transformer_hidden_dim, + attention_dim: attention_dim, + dropout: attention_dropout, + }, + }, + loss_fn: { + type: 'seqtag-nce-ranking-with-discrete-sampling', + reduction: 'mean', + log_key: 'seq_nce_loss', + num_samples: 100, + }, + initializer: { + regexes: [ + //[@'.*_feedforward._linear_layers.0.weight', {type: 'normal'}], + [@'.*feedforward._linear_layers.*weight', (if std.member(['tanh', 'sigmoid'], ff_activation) then { type: 'xavier_uniform', gain: gain } else { type: 'kaiming_uniform', nonlinearity: 'relu' })], + [@'.*linear_layers.*bias', { type: 'zero' }], + ], + }, + }, + data_loader: { + batch_sampler: { + type: 'bucket', + batch_size: 16, // effective batch size = batch_size*num_gradient_accumulation_steps + sorting_keys: ['tokens'], + }, + //max_instances_in_memory: if test == '1' then 10 else 1000, + }, + trainer: { + type: 'gradient_descent_minimax', + num_epochs: if test == '1' then 10 else 25, + num_gradient_accumulation_steps: 2, + grad_norm: { task_nn: 1.0 }, + patience: 4, + validation_metric: '+f1-measure-overall', + cuda_device: std.parseInt(cuda_device), + learning_rate_schedulers: { + task_nn: { + type: 'reduce_on_plateau', + factor: 0.5, + mode: 'max', + patience: 2, + verbose: true, + }, + }, + optimizer: { + optimizers: { + task_nn: + { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'huggingface_adamw', + }, + score_nn: { + lr: 0.00001, + weight_decay: ff_weight_decay, + type: 'adamw', + }, + }, + }, + checkpointer: { + keep_most_recent_by_count: 1, + }, + callbacks: [ + 'track_epoch_callback', + 'slurm', + ] + ( + if use_wandb then [ + { + type: 'wandb_allennlp', + should_log_parameter_statistics: false, + sub_callbacks: [{ type: 'log_best_validation_metrics', priority: 100 }], + save_model_archive: false, + watch_model: false, + }, + ] + else [] + ), + inner_mode: 'score_nn', + num_steps: { task_nn: 1, score_nn: 1 }, + }, +} diff --git a/structured_prediction_baselines/models/base.py b/structured_prediction_baselines/models/base.py index 1c19aa39..025286b5 100644 --- a/structured_prediction_baselines/models/base.py +++ b/structured_prediction_baselines/models/base.py @@ -42,10 +42,13 @@ logger = logging.getLogger(__name__) - +@Model.register( + "multi-task-score-based-learning-with-infnet", + constructor="from_partial_objects_with_shared_tasknn" +) @Model.register( "score-based-learning-with-infnet", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @Model.register("score-based-learning", constructor="from_partial_objects") class ScoreBasedLearningModel(LoggingMixin, Model): @@ -208,7 +211,7 @@ def parameters_for_model_mode( yield param @classmethod - def from_partial_objects_with_shared_tasknn( + def from_partial_objects_with_inference_module_as_sampler( cls, vocab: Vocabulary, loss_fn: Lazy[Loss], @@ -224,7 +227,8 @@ def from_partial_objects_with_shared_tasknn( ) -> "ScoreBasedLearningModel": """ This constructor is used only when the `sampler` is an instance of `SamplerContainer` - and we wish to use tasknn as both the inference_module and a sampler in the constituent sampler. + and we wish to use inference_module as a sampler in the constituent sampler i.e. share tasknn with both + infrence_module and sampler """ infnet_sampler = inference_module.construct( inference_nn=task_nn, @@ -296,6 +300,97 @@ def from_partial_objects_with_shared_tasknn( **kwargs, ) + @classmethod + def from_partial_objects_with_shared_tasknn( + cls, + vocab: Vocabulary, + loss_fn: Lazy[Loss], + inference_module: Lazy[Sampler], + task_nn: TaskNN, + sampler: Optional[Lazy[SamplerContainer]] = None, + score_nn: Optional[Lazy[ScoreNN]] = None, + oracle_value_function: Optional[OracleValueFunction] = None, + evaluation_module: Optional[Lazy[Sampler]] = None, + regularizer: Optional[RegularizerApplicator] = None, + initializer: Optional[InitializerApplicator] = None, + **kwargs: Any, + ) -> "ScoreBasedLearningModel": + """ + This constructor is used only when the `sampler` is an instance of `SamplerContainer` + and we wish to use inference_module as a sampler in the constituent sampler and share the tasknn among + the inference_module, sampler and scorenn thereby training the tasknn using `Multi-Task` objective. + """ + score_nn_ = score_nn.construct(task_nn=task_nn) + infnet_sampler = inference_module.construct( + inference_nn=task_nn, + score_nn=score_nn_, + oracle_value_function=oracle_value_function, + ) + + if oracle_value_function is not None: + if sampler is None: + sampler_ = AppendingSamplerContainer( + score_nn=score_nn_, + oracle_value_function=oracle_value_function, + constituent_samplers=[], + log_key="sampler", + ) + else: + sampler_ = sampler.construct( + score_nn=score_nn_, + oracle_value_function=oracle_value_function, + ) + loss_fn_ = loss_fn.construct( + score_nn=score_nn_, oracle_value_function=oracle_value_function + ) + else: + if sampler is None: + sampler_ = AppendingSamplerContainer( + score_nn=score_nn_, + constituent_samplers=[], + log_key="sampler", + ) + else: + + sampler_ = sampler.construct( + score_nn=score_nn_, + ) + loss_fn_ = loss_fn.construct( + score_nn=score_nn_, + ) + # add the infnet sampler + sampler_.append_sampler(infnet_sampler) + + # test-time inference. + # reconstruct the infnet sampler with shared tasknn weights + # to set it as the inference_module + inference_module_ = inference_module.construct( + inference_nn=task_nn, + score_nn=score_nn_, + oracle_value_function=oracle_value_function, + ) + inference_module_.log_key = inference_module_.log_key + "_inf" + + if evaluation_module is not None: + evaluation_module_ = evaluation_module.construct( + score_nn=score_nn_, oracle_value_function=oracle_value_function + ) + else: + evaluation_module_ = None + + return cls( + vocab=vocab, + sampler=sampler_, + loss_fn=loss_fn_, + oracle_value_function=oracle_value_function, + score_nn=score_nn_, + inference_module=inference_module_, + evaluation_module=evaluation_module_, + regularizer=regularizer, + initializer=initializer, + **kwargs, + ) + def calculate_metrics( self, x: Any, diff --git a/structured_prediction_baselines/models/multilabel_classification.py b/structured_prediction_baselines/models/multilabel_classification.py index 74e1c75b..7858d2af 100644 --- a/structured_prediction_baselines/models/multilabel_classification.py +++ b/structured_prediction_baselines/models/multilabel_classification.py @@ -25,14 +25,14 @@ @Model.register( "multi-label-classification-with-infnet", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @Model.register( "multi-label-classification", constructor="from_partial_objects" ) @ScoreBasedLearningModel.register( "multi-label-classification-with-infnet", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @ScoreBasedLearningModel.register( "multi-label-classification", constructor="from_partial_objects" @@ -104,7 +104,7 @@ def get_true_metrics(self, reset: bool = False) -> Dict[str, float]: ) @Model.register( "multi-label-classification-with-infnet-and-scorenn-evaluation", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) class MultilabelClassificationWithScoreNNEvaluation(MultilabelClassification): def __init__( diff --git a/structured_prediction_baselines/models/sequence_tagging.py b/structured_prediction_baselines/models/sequence_tagging.py index 46f11836..be7d2aba 100644 --- a/structured_prediction_baselines/models/sequence_tagging.py +++ b/structured_prediction_baselines/models/sequence_tagging.py @@ -282,9 +282,13 @@ def get_end_transitions(self) -> torch.Tensor: return end_transitions +@Model.register( + "seal-ner-multi-task", + constructor="from_partial_objects_with_shared_tasknn" +) @Model.register( "seal-ner", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @Model.register( "ner-seperate-inference-and-training-network", @@ -365,9 +369,13 @@ def get_metrics(self, reset: bool = False) -> Dict[str, float]: return metrics +@Model.register( + "seal-srl-multi-task", + constructor="from_partial_objects_with_shared_tasknn" +) @Model.register( "seal-srl", - constructor="from_partial_objects_with_shared_tasknn", + constructor="from_partial_objects_with_inference_module_as_sampler", ) @Model.register( "srl-seperate-inference-and-training-network", diff --git a/structured_prediction_baselines/modules/cnn_encoder.py b/structured_prediction_baselines/modules/cnn_encoder.py index 288f5c4d..9f58be52 100644 --- a/structured_prediction_baselines/modules/cnn_encoder.py +++ b/structured_prediction_baselines/modules/cnn_encoder.py @@ -62,6 +62,13 @@ def __init__( self.add_module("conv_layer_%d" % i, conv_layer) self._dropout = Dropout(dropout) + # TODO:: Fix maxpool_output_dim, Add pooling? + if output_dim: + self.projection_layer = Linear(maxpool_output_dim, output_dim) + self._output_dim = output_dim + else: + self.projection_layer = None + self._output_dim = maxpool_output_dim @overrides def get_input_dim(self) -> int: @@ -73,6 +80,7 @@ def get_output_dim(self) -> int: def forward(self, tokens: torch.Tensor, mask: torch.BoolTensor = None): if mask is not None: + # mask: (batch, seq_len) tokens = tokens * mask.unsqueeze(1).unsqueeze(-1) batch_size, n_samples, seq_length, _ = tokens.shape diff --git a/structured_prediction_baselines/modules/sampler/inference_net.py b/structured_prediction_baselines/modules/sampler/inference_net.py index 62bbd3fd..65f9900e 100644 --- a/structured_prediction_baselines/modules/sampler/inference_net.py +++ b/structured_prediction_baselines/modules/sampler/inference_net.py @@ -16,6 +16,7 @@ import numpy as np import torch from allennlp.common.lazy import Lazy +from allennlp.modules import FeedForward from allennlp.training.optimizers import Optimizer from structured_prediction_baselines.common import ModelMode from structured_prediction_baselines.modules.loss import Loss @@ -38,6 +39,8 @@ def parameters_with_model_mode( yield from self.inference_nn.parameters() if self.cost_augmented_layer is not None: yield from self.cost_augmented_layer.parameters() + if self.tag_projection_layer is not None: + yield from self.tag_projection_layer.parameters() def __init__( self, @@ -46,6 +49,7 @@ def __init__( score_nn: ScoreNN, cost_augmented_layer: Optional[CostAugmentedLayer] = None, oracle_value_function: Optional[OracleValueFunction] = None, + tag_projection_layer: Optional[FeedForward] = None, **kwargs: Any, ): assert ScoreNN is not None @@ -57,6 +61,7 @@ def __init__( self.inference_nn = inference_nn self.cost_augmented_layer = cost_augmented_layer self.loss_fn = loss_fn + self.tag_projection_layer = tag_projection_layer self.logging_children.append(self.loss_fn) @@ -113,9 +118,14 @@ def _get_values( **kwargs: Any, ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - y_inf: torch.Tensor = self.inference_nn(x, buffer).unsqueeze( - 1 - ) # (batch_size, 1, ...) unormalized + if self.tag_projection_layer: + y_inf: torch.Tensor = self.tag_projection_layer( + self.inference_nn(x, buffer) + ).unsqueeze(1) # (batch_size, 1, ...) unormalized + else: + y_inf: torch.Tensor = self.inference_nn(x, buffer).unsqueeze( + 1 + ) # (batch_size, 1, ...) unormalized # inference_nn is TaskNN so it will output tensor of shape (batch, ...) # hence the unsqueeze diff --git a/structured_prediction_baselines/modules/score_nn.py b/structured_prediction_baselines/modules/score_nn.py index 341d776a..835e9dae 100644 --- a/structured_prediction_baselines/modules/score_nn.py +++ b/structured_prediction_baselines/modules/score_nn.py @@ -1,6 +1,8 @@ from typing import List, Tuple, Union, Dict, Any, Optional from allennlp.common.registrable import Registrable import torch +from allennlp.modules import FeedForward + from .task_nn import TaskNN from .structured_score.structured_score import StructuredScore @@ -12,10 +14,14 @@ def __init__( self, task_nn: TaskNN, # (batch, ...) global_score: Optional[StructuredScore] = None, + residual_x: bool = False, + tag_projection_layer: Optional[FeedForward] = None, **kwargs: Any, ): super().__init__() # type:ignore self.task_nn = task_nn + self.tag_projection_layer = tag_projection_layer + self.residual_x = residual_x self.global_score = global_score self._dtype = self.compute_input_dtype() diff --git a/structured_prediction_baselines/modules/sequence_tagging_score_nn.py b/structured_prediction_baselines/modules/sequence_tagging_score_nn.py index 2680b33d..4c7fb728 100644 --- a/structured_prediction_baselines/modules/sequence_tagging_score_nn.py +++ b/structured_prediction_baselines/modules/sequence_tagging_score_nn.py @@ -25,10 +25,14 @@ def compute_local_score( # type:ignore # x, buffer # ) # (batch, ...) of unormalized logits # buffer["y_local"] = y_local - - y_local = self.task_nn( - x, buffer - ) # (batch, ...) of unormalized logits + if self.tag_projection_layer: + y_local = self.tag_projection_layer( + self.task_nn(x, buffer) + ) # (batch, ...) of unormalized logits + else: + y_local = self.task_nn( + x, buffer + ) # (batch, ...) of unormalized logits mask = buffer.get("mask") mask = mask.unsqueeze(1) @@ -54,7 +58,12 @@ def forward( score = None local_score = self.compute_local_score(x, y, buffer=buffer) - + if self.residual_x: + _, n_samples, _, _ = y.shape + embedded_x: torch.Tensor = buffer["embedded_x"] + assert len(embedded_x.shape) == 3 + embedded_x = embedded_x.unsqueeze(dim=1).repeat(1, n_samples, 1, 1) + y = torch.cat([y, embedded_x], dim=-1) global_score = self.compute_global_score(y, buffer) return local_score + global_score diff --git a/structured_prediction_baselines/modules/sequence_tagging_task_nn.py b/structured_prediction_baselines/modules/sequence_tagging_task_nn.py index 1cb7f7ca..d7e2b922 100644 --- a/structured_prediction_baselines/modules/sequence_tagging_task_nn.py +++ b/structured_prediction_baselines/modules/sequence_tagging_task_nn.py @@ -1,3 +1,5 @@ +import logging + from .task_nn import TaskNN, CostAugmentedLayer from typing import List, Tuple, Union, Dict, Any, Optional import torch @@ -12,6 +14,8 @@ import torch.nn.functional as F import allennlp.nn.util as util +logger = logging.getLogger(__name__) + @TaskNN.register("sequence-tagging") class SequenceTaggingTaskNN(TaskNN): @@ -23,6 +27,8 @@ def __init__( feedforward: Optional[FeedForward] = None, dropout: float = 0, label_namespace: str = "labels", + project_onto_tags: bool = True + ): """ @@ -50,10 +56,13 @@ def __init__( if output_dim is None: raise ValueError("output_dim cannot be None") - self.tag_projection_layer = TimeDistributed( - Linear(output_dim, self.num_tags) - ) # equivalent to Uj.b(x,t) in eq (3) - + if project_onto_tags: + self.tag_projection_layer = TimeDistributed( + Linear(output_dim, self.num_tags) + ) # equivalent to Uj.b(x,t) in eq (3) + else: + self.tag_projection_layer = None + logger.warning("Tag Projection Layer set to None. TaskNN would return encoded text instead of logits.") if dropout: self.dropout: Optional[torch.nn.Module] = torch.nn.Dropout(dropout) else: @@ -71,7 +80,7 @@ def forward( buffer["mask"] = mask embedded_text_input = self.text_field_embedder(tokens) - + buffer["embedded_x"] = embedded_text_input if self.encoder: encoded_text = self.encoder(embedded_text_input, mask) else: @@ -83,11 +92,14 @@ def forward( if self.feedforward: encoded_text = self.feedforward(encoded_text) - logits = self.tag_projection_layer(encoded_text) + if self.tag_projection_layer: + logits = self.tag_projection_layer(encoded_text) + + return ( + logits # shape (batch, sequence, num_tags) of unormalized logits + ) - return ( - logits # shape (batch, sequence, num_tags) of unormalized logits - ) + return encoded_text # shape (batch, sequence, output_dim) @CostAugmentedLayer.register("sequence-tagging-stacked") diff --git a/structured_prediction_baselines/modules/structured_score/sequence_tagging/cnn.py b/structured_prediction_baselines/modules/structured_score/sequence_tagging/cnn.py index 70c61625..efec98ac 100644 --- a/structured_prediction_baselines/modules/structured_score/sequence_tagging/cnn.py +++ b/structured_prediction_baselines/modules/structured_score/sequence_tagging/cnn.py @@ -1,4 +1,7 @@ from typing import List, Tuple, Union, Dict, Any, Optional + +from allennlp.nn import Activation + from structured_prediction_baselines.modules.structured_score.structured_score import StructuredScore import torch from structured_prediction_baselines.modules.cnn_encoder import Cnn2dEncoder @@ -6,13 +9,25 @@ @StructuredScore.register("cnn") class CNN(StructuredScore): - def __init__(self, num_tags: int, **kwargs: Any): - """ - TODO: Change kwargs to take hidden size and output size - """ + + def __init__(self, + num_tags: int, + embedding_dim: int, + num_filters: int, + ngram_filter_sizes: Tuple[int, ...], + dropout: float = 0.1, + conv_layer_activation: Activation = None, + output_dim: Optional[int] = None, + **kwargs: Any): super().__init__() self.num_tags = num_tags - self.encoder = Cnn2dEncoder(num_tags, embedding_dim=1, num_filters=50, ngram_filter_sizes=(3,), dropout=0.1) + self.encoder = Cnn2dEncoder(num_tags=num_tags, + embedding_dim=embedding_dim, + num_filters=num_filters, + ngram_filter_sizes=ngram_filter_sizes, + dropout=dropout, + conv_layer_activation=conv_layer_activation, + output_dim=output_dim) def forward( self, @@ -20,7 +35,7 @@ def forward( buffer: Dict, **kwargs: Any, ) -> torch.Tensor: - mask = buffer["mask"] + mask = buffer["mask"] # (batch, seq_len) output = self.encoder(y, mask) # (batch_size, num_samples or 1, ...) output = output.sum(dim=-1) # (batch_size, num_samples or 1, seq_length) output = output * mask.unsqueeze(1) diff --git a/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py b/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py index 6ce11838..7f883229 100644 --- a/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py +++ b/structured_prediction_baselines/modules/structured_score/sequence_tagging/self_attention.py @@ -11,6 +11,7 @@ class SelfAttention(StructuredScore): def __init__(self, num_tags: int, + input_dim: int = None, reduction: str = "max", M: int = 0, num_heads: int = 1, @@ -25,11 +26,12 @@ def __init__(self, self.reduction = reduction self.M = M assert self.M >= 0 - self.attention_dim = attention_dim or num_tags + self.input_dim = input_dim or num_tags + self.attention_dim = attention_dim or self.input_dim self.values_dim = values_dim or self.attention_dim self.attention_layer = SelfAttentionEncoder( num_heads, - input_dim=num_tags, + input_dim=self.input_dim, attention_dim=self.attention_dim, values_dim=self.values_dim, output_projection_dim=output_dim, @@ -86,8 +88,10 @@ def _get_attention_mask(self, n_samples, mask: torch.Tensor): @StructuredScore.register("self-attention-full-sequence") class SelfAttentionFullSequence(SelfAttention): - def __init__(self, num_tags: int, reduction: str = "max", **kwargs: Any): - super().__init__(num_tags, reduction, **kwargs) + def __init__(self, num_tags: int, + reduction: str = "max", + **kwargs: Any): + super().__init__(num_tags=num_tags, reduction=reduction, **kwargs) def _get_attention_mask(self, n_samples: int, mask: torch.Tensor): batch_size, seq_length = mask.shape diff --git a/sweep_configs/sequence_tagging/conll2003_bilstm_tasknn.yaml b/sweep_configs/sequence_tagging/conll2003_bilstm_tasknn.yaml new file mode 100644 index 00000000..7e9f41f4 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_bilstm_tasknn.yaml @@ -0,0 +1,25 @@ +name: conll2003_bilstm_tasknn +description: "Train tasknn using cross-entropy." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/conll2003_tasknn_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_nce_discrete_cnn_score.yaml b/sweep_configs/sequence_tagging/conll2003_nce_discrete_cnn_score.yaml new file mode 100644 index 00000000..33b7834c --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_nce_discrete_cnn_score.yaml @@ -0,0 +1,47 @@ +name: conll2003_nce_discrete_minus_tasknn_reverse +description: "Train tasknn using cross-entropy and score loss (v(f(x),y)). The score-nn will be trained using NCE with a - sign (score-ln Pn). The samples are taken as discrete samples from the tasknn output." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/conll2003_seal_nce_cnnscore_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.dvn_score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + # distribution: q_uniform + # min: 0 + # max: 11.99 + # q: 3 + model.loss_fn.num_samples: + value: 100 + # distribution: q_uniform + # q: 20 # 10, 25, ..., 50 + # min: 20 # ln(10) + # max: 100 # ln(50) \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_nce_discrete_linearchain_score.yaml b/sweep_configs/sequence_tagging/conll2003_nce_discrete_linearchain_score.yaml new file mode 100644 index 00000000..e194e229 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_nce_discrete_linearchain_score.yaml @@ -0,0 +1,47 @@ +name: conll2003_nce_discrete_minus_tasknn_reverse +description: "Train tasknn using cross-entropy and score loss (v(f(x),y)). The score-nn will be trained using NCE with a - sign (score-ln Pn). The samples are taken as discrete samples from the tasknn output." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/conll2003_seal_nce_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.dvn_score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + # distribution: q_uniform + # min: 0 + # max: 11.99 + # q: 3 + model.loss_fn.num_samples: + value: 100 + # distribution: q_uniform + # q: 20 # 10, 25, ..., 50 + # min: 20 # ln(10) + # max: 100 # ln(50) \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.yaml new file mode 100644 index 00000000..a55d4104 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.yaml @@ -0,0 +1,48 @@ +name: conll2003_seal_nce_discrete_self_attention_bilstm +description: "Train tasknn using cross-entropy and score loss (v(f(x),y)). The score-nn will be trained using NCE with a - sign (score-ln Pn). The samples are taken as discrete samples from the tasknn output." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.dvn_score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + model.loss_fn.num_samples: + value: 100 + env.attention_dim: + distribution: q_uniform + min: 49 # 50 + max: 249 # 200 + q: 50 + env.attention_dropout_10x: + distribution: q_uniform + min: 0.5 # 1 + max: 5.49 # 5 \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml new file mode 100644 index 00000000..d2491cef --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.yaml @@ -0,0 +1,49 @@ +name: conll2003_seal_nce_discrete_self_attention_bilstm_multi_task +program: allennlp +command: +- ${program} +- train-with-wandb +- model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_multi_task.jsonnet +- --include-package=structured_prediction_baselines +- --wandb-tags="mode=multi_task,task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.ff_weight_decay: + value: 1e-5 + env.score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + env.task_nn_lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + env.score_nn_lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + model.loss_fn.num_samples: + value: 100 + env.attention_dim: + distribution: q_uniform + min: 49 # 50 + max: 249 # 200 + q: 50 + env.attention_dropout_10x: + distribution: q_uniform + min: 0.5 # 1 + max: 5.49 # 5 \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.yaml b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.yaml new file mode 100644 index 00000000..57299f23 --- /dev/null +++ b/sweep_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.yaml @@ -0,0 +1,49 @@ +name: conll2003_seal_nce_discrete_self_attention_bilstm_residual +program: allennlp +command: +- ${program} +- train-with-wandb +- model_configs/sequence_tagging/conll2003_seal_nce_self_attention_bilstm_residual.jsonnet +- --include-package=structured_prediction_baselines +- --wandb-tags="mode=residual,task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=conll2003,inference_module=inference_net,inference_module=tasknn,structured_score=self_attention" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.cross_entropy_loss_weight: + value: 1.0 + env.ff_weight_decay: + value: 1e-5 + env.score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + env.tasknn_lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + env.scorenn_lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + model.loss_fn.num_samples: + value: 100 + env.attention_dim: + distribution: q_uniform + min: 49 # 50 + max: 249 # 200 + q: 50 + env.attention_dropout_10x: + distribution: q_uniform + min: 0.5 # 1 + max: 5.49 # 5 \ No newline at end of file diff --git a/sweep_configs/sequence_tagging/gendata_nce_discrete_minus_tasknn_reverse.yaml b/sweep_configs/sequence_tagging/gendata_nce_discrete_minus_tasknn_reverse.yaml new file mode 100644 index 00000000..2303d2db --- /dev/null +++ b/sweep_configs/sequence_tagging/gendata_nce_discrete_minus_tasknn_reverse.yaml @@ -0,0 +1,49 @@ +name: [data_name]_nce_discrete_minus_tasknn_reverse +description: "Train tasknn using cross-entropy and score loss (v(f(x),y)). The score-nn will be trained using NCE with a - sign (score-ln Pn). The samples are taken as discrete samples from the tasknn output." +program: allennlp +command: +- ${program} +- train_with_wandb +- model_configs/sequence_tagging/gendata_seal_nce_bilstm.jsonnet +- --include-package=structured_prediction_baselines +- --wandb_tags="task=seqtag,model=nce,sampler=nce_discrete_samples,dataset=[data_name],inference_module=inference_net,inference_module=tasknn" +- ${args} +- --file-friendly-logging +method: bayes +metric: + goal: maximize + name: "validation/best_f1-measure-overall" + +early_terminate: + type: hyperband + min_iter: 20 + +parameters: + env.dataset_name: + value: '[data_name]' + env.cross_entropy_loss_weight: + value: 1.0 + env.dvn_score_loss_weight: + distribution: log_uniform + min: -6.9 + max: 2.3 + trainer.optimizer.optimizers.task_nn.lr: + distribution: log_uniform + min: -12.5 + max: -4.5 + trainer.optimizer.optimizers.score_nn.lr: + distribution: log_uniform + min: -11.5 + max: -4.5 + env.score_nn_steps: + value: 1 + # distribution: q_uniform + # min: 0 + # max: 11.99 + # q: 3 + model.loss_fn.num_samples: + value: 100 + # distribution: q_uniform + # q: 20 # 10, 25, ..., 50 + # min: 20 # ln(10) + # max: 100 # ln(50) \ No newline at end of file diff --git a/tests/end2end/conftest.py b/tests/end2end/conftest.py index 5811606f..5cf24f5f 100644 --- a/tests/end2end/conftest.py +++ b/tests/end2end/conftest.py @@ -1,4 +1,10 @@ from .common import marks +import nltk + +try: + nltk.data.find('corpora/omw-1.4') +except LookupError: + nltk.download('omw-1.4') def pytest_configure(config) -> None: # type: ignore