From c0e2993947094e16b2c072d9e60a2b767d0743a6 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 20 Aug 2021 01:05:35 -0400 Subject: [PATCH 01/19] [CODE] Transfer from #475 + new structure --- textattack/metrics/__init__.py | 5 ++ textattack/metrics/attack_metrics/__init__.py | 14 ++++ .../metrics/attack_metrics/attack_metric.py | 25 +++++++ .../metrics/attack_metrics/attack_queries.py | 36 ++++++++++ .../attack_metrics/attack_success_rate.py | 69 +++++++++++++++++++ .../metrics/attack_metrics/words_perturbed.py | 65 +++++++++++++++++ .../metrics/quality_metrics/__init__.py | 12 ++++ .../metrics/quality_metrics/quality_metric.py | 0 8 files changed, 226 insertions(+) create mode 100644 textattack/metrics/__init__.py create mode 100644 textattack/metrics/attack_metrics/__init__.py create mode 100644 textattack/metrics/attack_metrics/attack_metric.py create mode 100644 textattack/metrics/attack_metrics/attack_queries.py create mode 100644 textattack/metrics/attack_metrics/attack_success_rate.py create mode 100644 textattack/metrics/attack_metrics/words_perturbed.py create mode 100644 textattack/metrics/quality_metrics/__init__.py create mode 100644 textattack/metrics/quality_metrics/quality_metric.py diff --git a/textattack/metrics/__init__.py b/textattack/metrics/__init__.py new file mode 100644 index 000000000..3711fccae --- /dev/null +++ b/textattack/metrics/__init__.py @@ -0,0 +1,5 @@ +""" +""" + +from .attack_metrics import AttackMetric +# from .quality_metrics import QualityMetric \ No newline at end of file diff --git a/textattack/metrics/attack_metrics/__init__.py b/textattack/metrics/attack_metrics/__init__.py new file mode 100644 index 000000000..d7f9b54ed --- /dev/null +++ b/textattack/metrics/attack_metrics/__init__.py @@ -0,0 +1,14 @@ +""" + +attack_metrics: +====================== + +TextAttack allows users to use their own metrics on adversarial examples or select common metrics to display. + + +""" + +from .attack_metric import AttackMetric +from .attack_queries import AttackQueries +from .attack_success_rate import AttackSuccessRate +from .words_perturbed import WordsPerturbed diff --git a/textattack/metrics/attack_metrics/attack_metric.py b/textattack/metrics/attack_metrics/attack_metric.py new file mode 100644 index 000000000..7f4bb03bb --- /dev/null +++ b/textattack/metrics/attack_metrics/attack_metric.py @@ -0,0 +1,25 @@ +""" +Attack Metrics Class +======================== + +""" + +from abc import ABC, abstractmethod + +from textattack.attack_results import AttackResult + + +class AttackMetric: + """A metric for evaluating Adversarial Attack candidates.""" + + @abstractmethod + def __init__(self, results, **kwargs): + """Creates pre-built :class:`~textattack.AttackMetric` that correspond to + evaluation metrics for adversarial examples. + """ + raise NotImplementedError() + + @abstractmethod + def calculate(): + """ Abstract function for computing any values which are to be calculated as a whole during initialization""" + raise NotImplementedError diff --git a/textattack/metrics/attack_metrics/attack_queries.py b/textattack/metrics/attack_metrics/attack_queries.py new file mode 100644 index 000000000..906470f60 --- /dev/null +++ b/textattack/metrics/attack_metrics/attack_queries.py @@ -0,0 +1,36 @@ +import numpy as np + +from textattack.attack_results import SkippedAttackResult + +from .attack_metric import AttackMetric + + +class AttackQueries(AttackMetric): + """Calculates all metrics related to number of queries in an attack + + Args: + results (:obj::`list`:class:`~textattack.goal_function_results.GoalFunctionResult`): + Attack results for each instance in dataset + """ + + def __init__(self, results): + self.results = results + + self.all_metrics = {} + + def calculate(self): + self.num_queries = np.array( + [ + r.num_queries + for r in self.results + if not isinstance(r, SkippedAttackResult) + ] + ) + self.all_metrics['avg_num_queries'] = self.avg_num_queries() + + return self.all_metrics + + def avg_num_queries(self): + avg_num_queries = self.num_queries.mean() + avg_num_queries = round(avg_num_queries, 2) + return avg_num_queries diff --git a/textattack/metrics/attack_metrics/attack_success_rate.py b/textattack/metrics/attack_metrics/attack_success_rate.py new file mode 100644 index 000000000..b884a59ef --- /dev/null +++ b/textattack/metrics/attack_metrics/attack_success_rate.py @@ -0,0 +1,69 @@ +from textattack.attack_results import FailedAttackResult, SkippedAttackResult + +from .attack_metric import AttackMetric + + +class AttackSuccessRate(AttackMetric): + """Calculates all metrics related to number of succesful, failed and skipped results in an attack + + Args: + results (:obj::`list`:class:`~textattack.goal_function_results.GoalFunctionResult`): + Attack results for each instance in dataset + """ + + def __init__(self, results): + self.results = results + self.failed_attacks = 0 + self.skipped_attacks = 0 + self.successful_attacks = 0 + self.total_attacks = len(self.results) + + self.all_metrics = {} + + def calculate(self): + for i, result in enumerate(self.results): + if isinstance(result, FailedAttackResult): + self.failed_attacks += 1 + continue + elif isinstance(result, SkippedAttackResult): + self.skipped_attacks += 1 + continue + else: + self.successful_attacks += 1 + + # Calculated numbers + self.all_metrics['successful_attacks'] = self.successful_attacks + self.all_metrics['failed_attacks'] = self.failed_attacks + self.all_metrics['skipped_attacks'] = self.skipped_attacks + + # Percentages wrt the calculations + self.all_metrics['original_accuracy'] = self.original_accuracy_perc() + self.all_metrics['attack_accuracy_perc'] = self.attack_accuracy_perc() + self.all_metrics['attack_success_rate'] = self.attack_success_rate_perc() + + return self.all_metrics + + + def original_accuracy_perc(self): + original_accuracy = ( + (self.total_attacks - self.skipped_attacks) * 100.0 / (self.total_attacks) + ) + original_accuracy = round(original_accuracy, 2) + return original_accuracy + + def attack_accuracy_perc(self): + accuracy_under_attack = (self.failed_attacks) * 100.0 / (self.total_attacks) + accuracy_under_attack = round(accuracy_under_attack, 2) + return accuracy_under_attack + + def attack_success_rate_perc(self): + if self.successful_attacks + self.failed_attacks == 0: + attack_success_rate = 0 + else: + attack_success_rate = ( + self.successful_attacks + * 100.0 + / (self.successful_attacks + self.failed_attacks) + ) + attack_success_rate = round(attack_success_rate, 2) + return attack_success_rate diff --git a/textattack/metrics/attack_metrics/words_perturbed.py b/textattack/metrics/attack_metrics/words_perturbed.py new file mode 100644 index 000000000..3043a53fa --- /dev/null +++ b/textattack/metrics/attack_metrics/words_perturbed.py @@ -0,0 +1,65 @@ +import numpy as np + +from textattack.attack_results import FailedAttackResult, SkippedAttackResult + +from .attack_metric import AttackMetric + + +class WordsPerturbed(AttackMetric): + def __init__(self, results): + self.results = results + self.total_attacks = len(self.results) + self.all_num_words = np.zeros(len(self.results)) + self.perturbed_word_percentages = np.zeros(len(self.results)) + self.num_words_changed_until_success = np.zeros(2 ** 16) + self.all_metrics = {} + + def calculate(self): + self.max_words_changed = 0 + for i, result in enumerate(self.results): + self.all_num_words[i] = len(result.original_result.attacked_text.words) + + if isinstance(result, FailedAttackResult) or isinstance( + result, SkippedAttackResult + ): + continue + + num_words_changed = len( + result.original_result.attacked_text.all_words_diff( + result.perturbed_result.attacked_text + ) + ) + self.num_words_changed_until_success[num_words_changed - 1] += 1 + self.max_words_changed = max( + self.max_words_changed or num_words_changed, num_words_changed + ) + if len(result.original_result.attacked_text.words) > 0: + perturbed_word_percentage = ( + num_words_changed + * 100.0 + / len(result.original_result.attacked_text.words) + ) + else: + perturbed_word_percentage = 0 + + self.perturbed_word_percentages[i] = perturbed_word_percentage + + self.all_metrics['avg_word_perturbed'] = self.avg_number_word_perturbed_num() + self.all_metrics['avg_word_perturbed_perc'] = self.avg_perturbation_perc() + self.all_metrics['max_words_changed'] = self.max_words_changed + self.all_metrics['num_words_changed_until_success'] = self.num_words_changed_until_success + + return self.all_metrics + + def avg_number_word_perturbed_num(self): + average_num_words = self.all_num_words.mean() + average_num_words = round(average_num_words, 2) + return average_num_words + + def avg_perturbation_perc(self): + self.perturbed_word_percentages = self.perturbed_word_percentages[ + self.perturbed_word_percentages > 0 + ] + average_perc_words_perturbed = self.perturbed_word_percentages.mean() + average_perc_words_perturbed = round(average_perc_words_perturbed, 2) + return average_perc_words_perturbed diff --git a/textattack/metrics/quality_metrics/__init__.py b/textattack/metrics/quality_metrics/__init__.py new file mode 100644 index 000000000..e01f34125 --- /dev/null +++ b/textattack/metrics/quality_metrics/__init__.py @@ -0,0 +1,12 @@ +""" + +attack_metrics: +====================== + +TextAttack allows users to use their own metrics on adversarial examples or select common metrics to display. + + +""" + +from .quality_metric import QualityMetric + diff --git a/textattack/metrics/quality_metrics/quality_metric.py b/textattack/metrics/quality_metrics/quality_metric.py new file mode 100644 index 000000000..e69de29bb From da4ca68520fe2d4e7970da4217b677fb7b2e9b1a Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 20 Aug 2021 01:21:29 -0400 Subject: [PATCH 02/19] [CODE] Changing metric to resemble constraints structure --- textattack/metrics/__init__.py | 5 ++++- textattack/metrics/attack_metrics/__init__.py | 1 - textattack/metrics/attack_metrics/attack_queries.py | 4 ++-- textattack/metrics/attack_metrics/attack_success_rate.py | 4 ++-- textattack/metrics/attack_metrics/words_perturbed.py | 4 ++-- .../metrics/{attack_metrics/attack_metric.py => metric.py} | 6 ++---- 6 files changed, 12 insertions(+), 12 deletions(-) rename textattack/metrics/{attack_metrics/attack_metric.py => metric.py} (86%) diff --git a/textattack/metrics/__init__.py b/textattack/metrics/__init__.py index 3711fccae..9cc1c3a48 100644 --- a/textattack/metrics/__init__.py +++ b/textattack/metrics/__init__.py @@ -1,5 +1,8 @@ """ """ -from .attack_metrics import AttackMetric +from .metric import Metric +from .attack_metrics import AttackSuccessRate +from .attack_metrics import WordsPerturbed +from .attack_metrics import AttackQueries # from .quality_metrics import QualityMetric \ No newline at end of file diff --git a/textattack/metrics/attack_metrics/__init__.py b/textattack/metrics/attack_metrics/__init__.py index d7f9b54ed..1cce96dcb 100644 --- a/textattack/metrics/attack_metrics/__init__.py +++ b/textattack/metrics/attack_metrics/__init__.py @@ -8,7 +8,6 @@ """ -from .attack_metric import AttackMetric from .attack_queries import AttackQueries from .attack_success_rate import AttackSuccessRate from .words_perturbed import WordsPerturbed diff --git a/textattack/metrics/attack_metrics/attack_queries.py b/textattack/metrics/attack_metrics/attack_queries.py index 906470f60..9e824fa0b 100644 --- a/textattack/metrics/attack_metrics/attack_queries.py +++ b/textattack/metrics/attack_metrics/attack_queries.py @@ -2,10 +2,10 @@ from textattack.attack_results import SkippedAttackResult -from .attack_metric import AttackMetric +from textattack.metrics import Metric -class AttackQueries(AttackMetric): +class AttackQueries(Metric): """Calculates all metrics related to number of queries in an attack Args: diff --git a/textattack/metrics/attack_metrics/attack_success_rate.py b/textattack/metrics/attack_metrics/attack_success_rate.py index b884a59ef..26d5c09e9 100644 --- a/textattack/metrics/attack_metrics/attack_success_rate.py +++ b/textattack/metrics/attack_metrics/attack_success_rate.py @@ -1,9 +1,9 @@ from textattack.attack_results import FailedAttackResult, SkippedAttackResult -from .attack_metric import AttackMetric +from textattack.metrics import Metric -class AttackSuccessRate(AttackMetric): +class AttackSuccessRate(Metric): """Calculates all metrics related to number of succesful, failed and skipped results in an attack Args: diff --git a/textattack/metrics/attack_metrics/words_perturbed.py b/textattack/metrics/attack_metrics/words_perturbed.py index 3043a53fa..0e8fc203e 100644 --- a/textattack/metrics/attack_metrics/words_perturbed.py +++ b/textattack/metrics/attack_metrics/words_perturbed.py @@ -2,10 +2,10 @@ from textattack.attack_results import FailedAttackResult, SkippedAttackResult -from .attack_metric import AttackMetric +from textattack.metrics import Metric -class WordsPerturbed(AttackMetric): +class WordsPerturbed(Metric): def __init__(self, results): self.results = results self.total_attacks = len(self.results) diff --git a/textattack/metrics/attack_metrics/attack_metric.py b/textattack/metrics/metric.py similarity index 86% rename from textattack/metrics/attack_metrics/attack_metric.py rename to textattack/metrics/metric.py index 7f4bb03bb..d3a755ac6 100644 --- a/textattack/metrics/attack_metrics/attack_metric.py +++ b/textattack/metrics/metric.py @@ -1,15 +1,13 @@ """ -Attack Metrics Class +Metric Class ======================== """ from abc import ABC, abstractmethod -from textattack.attack_results import AttackResult - -class AttackMetric: +class Metric(ABC): """A metric for evaluating Adversarial Attack candidates.""" @abstractmethod From 46781b1d5399ddf0e64c627ce2f2a1a53846ea66 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 20 Aug 2021 02:16:29 -0400 Subject: [PATCH 03/19] [CODE] Add code for perplexity --- .../metrics/quality_metrics/__init__.py | 4 +- .../metrics/quality_metrics/perplexity.py | 93 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 textattack/metrics/quality_metrics/perplexity.py diff --git a/textattack/metrics/quality_metrics/__init__.py b/textattack/metrics/quality_metrics/__init__.py index e01f34125..66ecff768 100644 --- a/textattack/metrics/quality_metrics/__init__.py +++ b/textattack/metrics/quality_metrics/__init__.py @@ -1,6 +1,6 @@ """ -attack_metrics: +perplexity: ====================== TextAttack allows users to use their own metrics on adversarial examples or select common metrics to display. @@ -8,5 +8,5 @@ """ -from .quality_metric import QualityMetric +from .perplexity import Perplexity diff --git a/textattack/metrics/quality_metrics/perplexity.py b/textattack/metrics/quality_metrics/perplexity.py new file mode 100644 index 000000000..53af224c8 --- /dev/null +++ b/textattack/metrics/quality_metrics/perplexity.py @@ -0,0 +1,93 @@ +from textattack.attack_results import FailedAttackResult, SkippedAttackResult + +from textattack.metrics import Metric + +from transformers import AutoTokenizer +from transformers import GPT2Tokenizer, GPT2LMHeadModel +import tqdm +import re +import torch + +class Perplexity(Metric): + """Calculates average Perplexity on all successfull attacks using a pre-trained small GPT-2 model + + Args: + results (:obj::`list`:class:`~textattack.goal_function_results.GoalFunctionResult`): + Attack results for each instance in dataset + """ + + def __init__(self, results): + self.results = results + self.all_metrics = {} + self.original_candidates = [] + self.successful_candidates = [] + self.ppl_model = GPT2LMHeadModel.from_pretrained('gpt2') + if torch.cuda.is_available(): + self.ppl_model.cuda() + self.ppl_tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + self.ppl_model.eval() + + self.original_candidates_ppl = [] + self.successful_candidates_ppl = [] + + def calculate(self): + + for i, result in enumerate(self.results): + if isinstance(result, FailedAttackResult): + continue + elif isinstance(result, SkippedAttackResult): + continue + else: + self.original_candidates.append(result.original_result.attacked_text.text.lower()) + self.successful_candidates.append(result.perturbed_result.attacked_text.text.lower()) + + + self.all_metrics['avg_original_perplexity'] = self.calc_ppl(self.original_candidates)[0] + self.all_metrics['avg_attack_perplexity'] = self.calc_ppl(self.successful_candidates)[0] + + + return self.all_metrics + + + def calc_ppl(self,texts): + eval_loss = 0 + ppl_losses = [] + nb_eval_steps = 0 + + with torch.no_grad(): + for text in texts: + text = self.process_string(text) + input_ids = torch.tensor(self.ppl_tokenizer.encode(text, add_special_tokens=True,truncation=True)) + if len(input_ids) < 2: + continue + if torch.cuda.is_available(): + self.input_ids.cuda() + outputs = self.ppl_model(input_ids, labels=input_ids) + lm_loss = outputs[0] + eval_loss += lm_loss.mean().item() + ppl_losses.append(torch.exp(torch.tensor(lm_loss.mean().item()))) + nb_eval_steps += 1 + + eval_loss = eval_loss / nb_eval_steps + perplexity = torch.exp(torch.tensor(eval_loss)) + + return perplexity.item(), ppl_losses + + def process_string(self,string): + string = re.sub("( )(\'[(m)(d)(t)(ll)(re)(ve)(s)])", r"\2", string) + string = re.sub("(\d+)( )([,\.])( )(\d+)", r"\1\3\5", string) + # U . S . -> U.S. + string = re.sub("(\w)( )(\.)( )(\w)( )(\.)", r"\1\3\5\7", string) + # reduce left space + string = re.sub("( )([,\.!?:;)])", r"\2", string) + # reduce right space + string = re.sub("([(])( )", r"\1", string) + string = re.sub("s '", "s'", string) + # reduce both space + string = re.sub("(')( )(\S+)( )(')", r"\1\3\5", string) + string = re.sub("(\")( )(\S+)( )(\")", r"\1\3\5", string) + string = re.sub("(\w+) (-+) (\w+)", r"\1\2\3", string) + string = re.sub("(\w+) (/+) (\w+)", r"\1\2\3", string) + # string = re.sub(" ' ", "'", string) + return string + From 5a882536925e0d60231c4d75b8e67e9a77962575 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 20 Aug 2021 02:18:43 -0400 Subject: [PATCH 04/19] [CODE] Fixing init file --- textattack/metrics/__init__.py | 4 +++- textattack/metrics/quality_metrics/quality_metric.py | 0 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 textattack/metrics/quality_metrics/quality_metric.py diff --git a/textattack/metrics/__init__.py b/textattack/metrics/__init__.py index 9cc1c3a48..004479cba 100644 --- a/textattack/metrics/__init__.py +++ b/textattack/metrics/__init__.py @@ -2,7 +2,9 @@ """ from .metric import Metric + from .attack_metrics import AttackSuccessRate from .attack_metrics import WordsPerturbed from .attack_metrics import AttackQueries -# from .quality_metrics import QualityMetric \ No newline at end of file + +from .quality_metrics import Perplexity \ No newline at end of file diff --git a/textattack/metrics/quality_metrics/quality_metric.py b/textattack/metrics/quality_metrics/quality_metric.py deleted file mode 100644 index e69de29bb..000000000 From 4a94be920944dbf3e86bd2380005a0c59fcdbc84 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 20 Aug 2021 10:11:14 -0400 Subject: [PATCH 05/19] [CODE] Add command-line option for quality metrics --- textattack/__init__.py | 17 +- textattack/attack_args.py | 9 ++ textattack/attacker.py | 8 + textattack/loggers/attack_log_manager.py | 146 +++++++----------- textattack/metrics/__init__.py | 2 +- .../metrics/attack_metrics/attack_queries.py | 3 +- .../attack_metrics/attack_success_rate.py | 14 +- .../metrics/attack_metrics/words_perturbed.py | 11 +- .../metrics/quality_metrics/__init__.py | 1 - .../metrics/quality_metrics/perplexity.py | 54 ++++--- 10 files changed, 128 insertions(+), 137 deletions(-) diff --git a/textattack/__init__.py b/textattack/__init__.py index 12f52331a..5aeaa97db 100644 --- a/textattack/__init__.py +++ b/textattack/__init__.py @@ -9,15 +9,6 @@ TextAttack provides components for common NLP tasks like sentence encoding, grammar-checking, and word replacement that can be used on their own. """ -from .attack_args import AttackArgs, CommandLineAttackArgs -from .augment_args import AugmenterArgs -from .dataset_args import DatasetArgs -from .model_args import ModelArgs -from .training_args import TrainingArgs, CommandLineTrainingArgs -from .attack import Attack -from .attacker import Attacker -from .trainer import Trainer - from . import ( attack_recipes, attack_results, @@ -33,5 +24,13 @@ shared, transformations, ) +from .attack import Attack +from .attack_args import AttackArgs, CommandLineAttackArgs +from .attacker import Attacker +from .augment_args import AugmenterArgs +from .dataset_args import DatasetArgs +from .model_args import ModelArgs +from .trainer import Trainer +from .training_args import CommandLineTrainingArgs, TrainingArgs name = "textattack" diff --git a/textattack/attack_args.py b/textattack/attack_args.py index 72f5bdec1..24f745f32 100644 --- a/textattack/attack_args.py +++ b/textattack/attack_args.py @@ -173,6 +173,8 @@ class AttackArgs: Disable displaying individual attack results to stdout. silent (:obj:`bool`, `optional`, defaults to :obj:`False`): Disable all logging (except for errors). This is stronger than :obj:`disable_stdout`. + enable_advance_metrics (:obj:`bool`, `optional`, defaults to :obj:`False`): + Enable calculation and display of optional advance post-hoc metrics like perplexity, grammar errors, etc. """ num_examples: int = 10 @@ -193,6 +195,7 @@ class AttackArgs: log_to_wandb: str = None disable_stdout: bool = False silent: bool = False + enable_advance_metrics: bool = False def __post_init__(self): if self.num_successful_examples: @@ -350,6 +353,12 @@ def _add_parser_args(cls, parser): default=default_obj.silent, help="Disable all logging", ) + parser.add_argument( + "--enable-advance-metrics", + action="store_true", + default=default_obj.enable_advance_metrics, + help="Enable advance metric calculations", + ) return parser diff --git a/textattack/attacker.py b/textattack/attacker.py index 7aa14a0ab..96a9e21cb 100644 --- a/textattack/attacker.py +++ b/textattack/attacker.py @@ -219,6 +219,10 @@ def _attack(self): # Enable summary stdout if not self.attack_args.silent and self.attack_args.disable_stdout: self.attack_log_manager.enable_stdout() + + if self.attack_args.enable_advance_metrics: + self.attack_log_manager.enable_advance_metrics = True + self.attack_log_manager.log_summary() self.attack_log_manager.flush() print() @@ -390,6 +394,10 @@ def _attack_parallel(self): # Enable summary stdout. if not self.attack_args.silent and self.attack_args.disable_stdout: self.attack_log_manager.enable_stdout() + + if self.attack_args.enable_advance_metrics: + self.attack_log_manager.enable_advance_metrics = True + self.attack_log_manager.log_summary() self.attack_log_manager.flush() print() diff --git a/textattack/loggers/attack_log_manager.py b/textattack/loggers/attack_log_manager.py index ab608b48e..9dcb85315 100644 --- a/textattack/loggers/attack_log_manager.py +++ b/textattack/loggers/attack_log_manager.py @@ -3,9 +3,12 @@ ======================== """ -import numpy as np - -from textattack.attack_results import FailedAttackResult, SkippedAttackResult +from textattack.metrics.attack_metrics import ( + AttackQueries, + AttackSuccessRate, + WordsPerturbed, +) +from textattack.metrics.quality_metrics import Perplexity from . import CSVLogger, FileLogger, VisdomLogger, WeightsAndBiasesLogger @@ -16,6 +19,7 @@ class AttackLogManager: def __init__(self): self.loggers = [] self.results = [] + self.enable_advance_metrics = False def enable_stdout(self): self.loggers.append(FileLogger(stdout=True)) @@ -72,103 +76,69 @@ def log_summary(self): total_attacks = len(self.results) if total_attacks == 0: return - # Count things about attacks. - all_num_words = np.zeros(len(self.results)) - perturbed_word_percentages = np.zeros(len(self.results)) - num_words_changed_until_success = np.zeros( - 2 ** 16 - ) # @ TODO: be smarter about this - failed_attacks = 0 - skipped_attacks = 0 - successful_attacks = 0 - max_words_changed = 0 - for i, result in enumerate(self.results): - all_num_words[i] = len(result.original_result.attacked_text.words) - if isinstance(result, FailedAttackResult): - failed_attacks += 1 - continue - elif isinstance(result, SkippedAttackResult): - skipped_attacks += 1 - continue - else: - successful_attacks += 1 - num_words_changed = result.original_result.attacked_text.words_diff_num( - result.perturbed_result.attacked_text - ) - # num_words_changed = len( - # result.original_result.attacked_text.all_words_diff( - # result.perturbed_result.attacked_text - # ) - # ) - num_words_changed_until_success[num_words_changed - 1] += 1 - max_words_changed = max( - max_words_changed or num_words_changed, num_words_changed - ) - if len(result.original_result.attacked_text.words) > 0: - perturbed_word_percentage = ( - num_words_changed - * 100.0 - / len(result.original_result.attacked_text.words) - ) - else: - perturbed_word_percentage = 0 - perturbed_word_percentages[i] = perturbed_word_percentage - - # Original classifier success rate on these samples. - original_accuracy = (total_attacks - skipped_attacks) * 100.0 / (total_attacks) - original_accuracy = str(round(original_accuracy, 2)) + "%" - - # New classifier success rate on these samples. - accuracy_under_attack = (failed_attacks) * 100.0 / (total_attacks) - accuracy_under_attack = str(round(accuracy_under_attack, 2)) + "%" - - # Attack success rate. - if successful_attacks + failed_attacks == 0: - attack_success_rate = 0 - else: - attack_success_rate = ( - successful_attacks * 100.0 / (successful_attacks + failed_attacks) - ) - attack_success_rate = str(round(attack_success_rate, 2)) + "%" - perturbed_word_percentages = perturbed_word_percentages[ - perturbed_word_percentages > 0 - ] - average_perc_words_perturbed = perturbed_word_percentages.mean() - average_perc_words_perturbed = str(round(average_perc_words_perturbed, 2)) + "%" - - average_num_words = all_num_words.mean() - average_num_words = str(round(average_num_words, 2)) + # Default metrics - calculated on every attack + attack_success_stats = AttackSuccessRate(self.results).calculate() + words_perturbed_stats = WordsPerturbed(self.results).calculate() + attack_query_stats = AttackQueries(self.results).calculate() + # @TODO generate this table based on user input - each column in specific class + # Example to demonstrate: + # summary_table_rows = attack_success_stats.display_row() + words_perturbed_stats.display_row() + ... summary_table_rows = [ - ["Number of successful attacks:", str(successful_attacks)], - ["Number of failed attacks:", str(failed_attacks)], - ["Number of skipped attacks:", str(skipped_attacks)], - ["Original accuracy:", original_accuracy], - ["Accuracy under attack:", accuracy_under_attack], - ["Attack success rate:", attack_success_rate], - ["Average perturbed word %:", average_perc_words_perturbed], - ["Average num. words per input:", average_num_words], + [ + "Number of successful attacks:", + attack_success_stats["successful_attacks"], + ], + ["Number of failed attacks:", attack_success_stats["failed_attacks"]], + ["Number of skipped attacks:", attack_success_stats["skipped_attacks"]], + [ + "Original accuracy:", + str(attack_success_stats["original_accuracy"]) + "%", + ], + [ + "Accuracy under attack:", + str(attack_success_stats["attack_accuracy_perc"]) + "%", + ], + [ + "Attack success rate:", + str(attack_success_stats["attack_success_rate"]) + "%", + ], + [ + "Average perturbed word %:", + str(words_perturbed_stats["avg_word_perturbed_perc"]) + "%", + ], + [ + "Average num. words per input:", + words_perturbed_stats["avg_word_perturbed"], + ], ] - num_queries = np.array( - [ - r.num_queries - for r in self.results - if not isinstance(r, SkippedAttackResult) - ] + summary_table_rows.append( + ["Avg num queries:", attack_query_stats["avg_num_queries"]] ) - avg_num_queries = num_queries.mean() - avg_num_queries = str(round(avg_num_queries, 2)) - summary_table_rows.append(["Avg num queries:", avg_num_queries]) + + if self.enable_advance_metrics: + perplexity_stats = Perplexity(self.results).calculate() + + summary_table_rows.append( + [ + "Avg Original Perplexity:", + perplexity_stats["avg_original_perplexity"], + ] + ) + summary_table_rows.append( + ["Avg Attack Perplexity:", perplexity_stats["avg_attack_perplexity"]] + ) + self.log_summary_rows( summary_table_rows, "Attack Results", "attack_results_summary" ) # Show histogram of words changed. - numbins = max(max_words_changed, 10) + numbins = max(words_perturbed_stats["max_words_changed"], 10) for logger in self.loggers: logger.log_hist( - num_words_changed_until_success[:numbins], + words_perturbed_stats["num_words_changed_until_success"][:numbins], numbins=numbins, title="Num Words Perturbed", window_id="num_words_perturbed", diff --git a/textattack/metrics/__init__.py b/textattack/metrics/__init__.py index 004479cba..b94b0f56a 100644 --- a/textattack/metrics/__init__.py +++ b/textattack/metrics/__init__.py @@ -7,4 +7,4 @@ from .attack_metrics import WordsPerturbed from .attack_metrics import AttackQueries -from .quality_metrics import Perplexity \ No newline at end of file +from .quality_metrics import Perplexity diff --git a/textattack/metrics/attack_metrics/attack_queries.py b/textattack/metrics/attack_metrics/attack_queries.py index 9e824fa0b..5c2658d39 100644 --- a/textattack/metrics/attack_metrics/attack_queries.py +++ b/textattack/metrics/attack_metrics/attack_queries.py @@ -1,7 +1,6 @@ import numpy as np from textattack.attack_results import SkippedAttackResult - from textattack.metrics import Metric @@ -26,7 +25,7 @@ def calculate(self): if not isinstance(r, SkippedAttackResult) ] ) - self.all_metrics['avg_num_queries'] = self.avg_num_queries() + self.all_metrics["avg_num_queries"] = self.avg_num_queries() return self.all_metrics diff --git a/textattack/metrics/attack_metrics/attack_success_rate.py b/textattack/metrics/attack_metrics/attack_success_rate.py index 26d5c09e9..6b26b0e0c 100644 --- a/textattack/metrics/attack_metrics/attack_success_rate.py +++ b/textattack/metrics/attack_metrics/attack_success_rate.py @@ -1,5 +1,4 @@ from textattack.attack_results import FailedAttackResult, SkippedAttackResult - from textattack.metrics import Metric @@ -32,18 +31,17 @@ def calculate(self): self.successful_attacks += 1 # Calculated numbers - self.all_metrics['successful_attacks'] = self.successful_attacks - self.all_metrics['failed_attacks'] = self.failed_attacks - self.all_metrics['skipped_attacks'] = self.skipped_attacks + self.all_metrics["successful_attacks"] = self.successful_attacks + self.all_metrics["failed_attacks"] = self.failed_attacks + self.all_metrics["skipped_attacks"] = self.skipped_attacks # Percentages wrt the calculations - self.all_metrics['original_accuracy'] = self.original_accuracy_perc() - self.all_metrics['attack_accuracy_perc'] = self.attack_accuracy_perc() - self.all_metrics['attack_success_rate'] = self.attack_success_rate_perc() + self.all_metrics["original_accuracy"] = self.original_accuracy_perc() + self.all_metrics["attack_accuracy_perc"] = self.attack_accuracy_perc() + self.all_metrics["attack_success_rate"] = self.attack_success_rate_perc() return self.all_metrics - def original_accuracy_perc(self): original_accuracy = ( (self.total_attacks - self.skipped_attacks) * 100.0 / (self.total_attacks) diff --git a/textattack/metrics/attack_metrics/words_perturbed.py b/textattack/metrics/attack_metrics/words_perturbed.py index 0e8fc203e..9558270d3 100644 --- a/textattack/metrics/attack_metrics/words_perturbed.py +++ b/textattack/metrics/attack_metrics/words_perturbed.py @@ -1,7 +1,6 @@ import numpy as np from textattack.attack_results import FailedAttackResult, SkippedAttackResult - from textattack.metrics import Metric @@ -44,10 +43,12 @@ def calculate(self): self.perturbed_word_percentages[i] = perturbed_word_percentage - self.all_metrics['avg_word_perturbed'] = self.avg_number_word_perturbed_num() - self.all_metrics['avg_word_perturbed_perc'] = self.avg_perturbation_perc() - self.all_metrics['max_words_changed'] = self.max_words_changed - self.all_metrics['num_words_changed_until_success'] = self.num_words_changed_until_success + self.all_metrics["avg_word_perturbed"] = self.avg_number_word_perturbed_num() + self.all_metrics["avg_word_perturbed_perc"] = self.avg_perturbation_perc() + self.all_metrics["max_words_changed"] = self.max_words_changed + self.all_metrics[ + "num_words_changed_until_success" + ] = self.num_words_changed_until_success return self.all_metrics diff --git a/textattack/metrics/quality_metrics/__init__.py b/textattack/metrics/quality_metrics/__init__.py index 66ecff768..6e4df2d54 100644 --- a/textattack/metrics/quality_metrics/__init__.py +++ b/textattack/metrics/quality_metrics/__init__.py @@ -9,4 +9,3 @@ """ from .perplexity import Perplexity - diff --git a/textattack/metrics/quality_metrics/perplexity.py b/textattack/metrics/quality_metrics/perplexity.py index 53af224c8..3e50ab5e2 100644 --- a/textattack/metrics/quality_metrics/perplexity.py +++ b/textattack/metrics/quality_metrics/perplexity.py @@ -1,12 +1,12 @@ -from textattack.attack_results import FailedAttackResult, SkippedAttackResult +import re +import torch +import tqdm +from transformers import AutoTokenizer, GPT2LMHeadModel, GPT2Tokenizer + +from textattack.attack_results import FailedAttackResult, SkippedAttackResult from textattack.metrics import Metric -from transformers import AutoTokenizer -from transformers import GPT2Tokenizer, GPT2LMHeadModel -import tqdm -import re -import torch class Perplexity(Metric): """Calculates average Perplexity on all successfull attacks using a pre-trained small GPT-2 model @@ -21,35 +21,40 @@ def __init__(self, results): self.all_metrics = {} self.original_candidates = [] self.successful_candidates = [] - self.ppl_model = GPT2LMHeadModel.from_pretrained('gpt2') + self.ppl_model = GPT2LMHeadModel.from_pretrained("gpt2") if torch.cuda.is_available(): self.ppl_model.cuda() - self.ppl_tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + self.ppl_tokenizer = GPT2Tokenizer.from_pretrained("gpt2") self.ppl_model.eval() self.original_candidates_ppl = [] self.successful_candidates_ppl = [] def calculate(self): - + for i, result in enumerate(self.results): if isinstance(result, FailedAttackResult): continue elif isinstance(result, SkippedAttackResult): continue else: - self.original_candidates.append(result.original_result.attacked_text.text.lower()) - self.successful_candidates.append(result.perturbed_result.attacked_text.text.lower()) - - - self.all_metrics['avg_original_perplexity'] = self.calc_ppl(self.original_candidates)[0] - self.all_metrics['avg_attack_perplexity'] = self.calc_ppl(self.successful_candidates)[0] - + self.original_candidates.append( + result.original_result.attacked_text.text.lower() + ) + self.successful_candidates.append( + result.perturbed_result.attacked_text.text.lower() + ) + + self.all_metrics["avg_original_perplexity"] = round( + self.calc_ppl(self.original_candidates)[0], 2 + ) + self.all_metrics["avg_attack_perplexity"] = round( + self.calc_ppl(self.successful_candidates)[0], 2 + ) return self.all_metrics - - def calc_ppl(self,texts): + def calc_ppl(self, texts): eval_loss = 0 ppl_losses = [] nb_eval_steps = 0 @@ -57,7 +62,11 @@ def calc_ppl(self,texts): with torch.no_grad(): for text in texts: text = self.process_string(text) - input_ids = torch.tensor(self.ppl_tokenizer.encode(text, add_special_tokens=True,truncation=True)) + input_ids = torch.tensor( + self.ppl_tokenizer.encode( + text, add_special_tokens=True, truncation=True + ) + ) if len(input_ids) < 2: continue if torch.cuda.is_available(): @@ -73,8 +82,8 @@ def calc_ppl(self,texts): return perplexity.item(), ppl_losses - def process_string(self,string): - string = re.sub("( )(\'[(m)(d)(t)(ll)(re)(ve)(s)])", r"\2", string) + def process_string(self, string): + string = re.sub("( )('[(m)(d)(t)(ll)(re)(ve)(s)])", r"\2", string) string = re.sub("(\d+)( )([,\.])( )(\d+)", r"\1\3\5", string) # U . S . -> U.S. string = re.sub("(\w)( )(\.)( )(\w)( )(\.)", r"\1\3\5\7", string) @@ -85,9 +94,8 @@ def process_string(self,string): string = re.sub("s '", "s'", string) # reduce both space string = re.sub("(')( )(\S+)( )(')", r"\1\3\5", string) - string = re.sub("(\")( )(\S+)( )(\")", r"\1\3\5", string) + string = re.sub('(")( )(\S+)( )(")', r"\1\3\5", string) string = re.sub("(\w+) (-+) (\w+)", r"\1\2\3", string) string = re.sub("(\w+) (/+) (\w+)", r"\1\2\3", string) # string = re.sub(" ' ", "'", string) return string - From 5e929f2498f582fc352bb15850367856ced21bd8 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 27 Aug 2021 00:40:23 -0400 Subject: [PATCH 06/19] [FIX] Import order+metrics import --- textattack/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/textattack/__init__.py b/textattack/__init__.py index 5aeaa97db..a169173eb 100644 --- a/textattack/__init__.py +++ b/textattack/__init__.py @@ -8,6 +8,15 @@ TextAttack provides components for common NLP tasks like sentence encoding, grammar-checking, and word replacement that can be used on their own. """ +from .attack_args import AttackArgs, CommandLineAttackArgs +from .augment_args import AugmenterArgs +from .dataset_args import DatasetArgs +from .model_args import ModelArgs +from .training_args import TrainingArgs, CommandLineTrainingArgs +from .attack import Attack +from .attacker import Attacker +from .trainer import Trainer +from .metrics import Metric from . import ( attack_recipes, @@ -19,18 +28,12 @@ goal_function_results, goal_functions, loggers, + metrics, models, search_methods, shared, transformations, ) -from .attack import Attack -from .attack_args import AttackArgs, CommandLineAttackArgs -from .attacker import Attacker -from .augment_args import AugmenterArgs -from .dataset_args import DatasetArgs -from .model_args import ModelArgs -from .trainer import Trainer -from .training_args import CommandLineTrainingArgs, TrainingArgs + name = "textattack" From a1b2c5bc7aef5a495dde4c67826be727788c97ed Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 27 Aug 2021 02:36:17 -0400 Subject: [PATCH 07/19] [CODE] New USE metric WIP Signed-off-by: sanchit97 --- textattack/metrics/__init__.py | 1 + .../metrics/quality_metrics/__init__.py | 1 + .../metrics/quality_metrics/perplexity.py | 78 +++++++++---------- textattack/metrics/quality_metrics/use.py | 54 +++++++++++++ 4 files changed, 92 insertions(+), 42 deletions(-) create mode 100644 textattack/metrics/quality_metrics/use.py diff --git a/textattack/metrics/__init__.py b/textattack/metrics/__init__.py index b94b0f56a..7bb7bd0bf 100644 --- a/textattack/metrics/__init__.py +++ b/textattack/metrics/__init__.py @@ -8,3 +8,4 @@ from .attack_metrics import AttackQueries from .quality_metrics import Perplexity +from .quality_metrics import USEMetric \ No newline at end of file diff --git a/textattack/metrics/quality_metrics/__init__.py b/textattack/metrics/quality_metrics/__init__.py index 6e4df2d54..79f707ffc 100644 --- a/textattack/metrics/quality_metrics/__init__.py +++ b/textattack/metrics/quality_metrics/__init__.py @@ -9,3 +9,4 @@ """ from .perplexity import Perplexity +from .use import USEMetric diff --git a/textattack/metrics/quality_metrics/perplexity.py b/textattack/metrics/quality_metrics/perplexity.py index 3e50ab5e2..41b3f757b 100644 --- a/textattack/metrics/quality_metrics/perplexity.py +++ b/textattack/metrics/quality_metrics/perplexity.py @@ -6,6 +6,7 @@ from textattack.attack_results import FailedAttackResult, SkippedAttackResult from textattack.metrics import Metric +import textattack.shared.utils class Perplexity(Metric): @@ -22,10 +23,11 @@ def __init__(self, results): self.original_candidates = [] self.successful_candidates = [] self.ppl_model = GPT2LMHeadModel.from_pretrained("gpt2") - if torch.cuda.is_available(): - self.ppl_model.cuda() + self.ppl_model.to(textattack.shared.utils.device) self.ppl_tokenizer = GPT2Tokenizer.from_pretrained("gpt2") self.ppl_model.eval() + self.max_length = self.ppl_model.config.n_positions + self.stride = 128 self.original_candidates_ppl = [] self.successful_candidates_ppl = [] @@ -45,57 +47,49 @@ def calculate(self): result.perturbed_result.attacked_text.text.lower() ) + ppl_orig = self.calc_ppl(self.original_candidates) + ppl_attack = self.calc_ppl(self.successful_candidates) + self.all_metrics["avg_original_perplexity"] = round( - self.calc_ppl(self.original_candidates)[0], 2 + ppl_orig[0], 2 ) + self.all_metrics["original_perplexity_list"] = ppl_orig[1] + self.all_metrics["avg_attack_perplexity"] = round( - self.calc_ppl(self.successful_candidates)[0], 2 + ppl_attack[0], 2 ) + self.all_metrics["attack_perplexity_list"] = ppl_attack[1] return self.all_metrics def calc_ppl(self, texts): - eval_loss = 0 - ppl_losses = [] - nb_eval_steps = 0 + + ppl_vals = [] with torch.no_grad(): for text in texts: - text = self.process_string(text) + eval_loss = [] input_ids = torch.tensor( self.ppl_tokenizer.encode( - text, add_special_tokens=True, truncation=True + text, add_special_tokens=True ) - ) - if len(input_ids) < 2: - continue - if torch.cuda.is_available(): - self.input_ids.cuda() - outputs = self.ppl_model(input_ids, labels=input_ids) - lm_loss = outputs[0] - eval_loss += lm_loss.mean().item() - ppl_losses.append(torch.exp(torch.tensor(lm_loss.mean().item()))) - nb_eval_steps += 1 - - eval_loss = eval_loss / nb_eval_steps - perplexity = torch.exp(torch.tensor(eval_loss)) - - return perplexity.item(), ppl_losses - - def process_string(self, string): - string = re.sub("( )('[(m)(d)(t)(ll)(re)(ve)(s)])", r"\2", string) - string = re.sub("(\d+)( )([,\.])( )(\d+)", r"\1\3\5", string) - # U . S . -> U.S. - string = re.sub("(\w)( )(\.)( )(\w)( )(\.)", r"\1\3\5\7", string) - # reduce left space - string = re.sub("( )([,\.!?:;)])", r"\2", string) - # reduce right space - string = re.sub("([(])( )", r"\1", string) - string = re.sub("s '", "s'", string) - # reduce both space - string = re.sub("(')( )(\S+)( )(')", r"\1\3\5", string) - string = re.sub('(")( )(\S+)( )(")', r"\1\3\5", string) - string = re.sub("(\w+) (-+) (\w+)", r"\1\2\3", string) - string = re.sub("(\w+) (/+) (\w+)", r"\1\2\3", string) - # string = re.sub(" ' ", "'", string) - return string + ).unsqueeze(0) + + for i in range(0, input_ids.size(1), self.stride): + begin_loc = max(i + self.stride - self.max_length, 0) + end_loc = min(i + self.stride, input_ids.size(1)) + trg_len = end_loc - i # may be different from stride on last loop + input_ids_t = input_ids[:,begin_loc:end_loc].to(textattack.shared.utils.device) + target_ids = input_ids_t.clone() + target_ids[:,:-trg_len] = -100 + + outputs = self.ppl_model(input_ids_t, labels=target_ids) + log_likelihood = outputs[0] * trg_len + + eval_loss.append(log_likelihood) + + ppl_vals.append(torch.exp(torch.stack(eval_loss).sum() / end_loc).item()) + + + return sum(ppl_vals)/len(ppl_vals), ppl_vals + diff --git a/textattack/metrics/quality_metrics/use.py b/textattack/metrics/quality_metrics/use.py new file mode 100644 index 000000000..c4e3d1ec9 --- /dev/null +++ b/textattack/metrics/quality_metrics/use.py @@ -0,0 +1,54 @@ +from textattack.attack_results import FailedAttackResult, SkippedAttackResult +from textattack.metrics import Metric +from textattack.constraints.semantics.sentence_encoders import SentenceEncoder +from textattack.shared.utils import LazyLoader + +hub = LazyLoader("tensorflow_hub", globals(), "tensorflow_hub") + + +class USEMetric(Metric): + """Constraint using similarity between sentence encodings of x and x_adv + where the text embeddings are created using the Universal Sentence + Encoder.""" + + def __init__(self, results, **kwargs): + self.results = results + self.use_obj = SentenceEncoder() + self.original_candidates = [] + self.successful_candidates = [] + + if kwargs["large"]: + tfhub_url = "https://tfhub.dev/google/universal-sentence-encoder-large/5" + else: + tfhub_url = "https://tfhub.dev/google/universal-sentence-encoder/4" + + self._tfhub_url = tfhub_url + # Lazily load the model + self.model = hub.load(self._tfhub_url) + + def calculate(self): + for i, result in enumerate(self.results): + if isinstance(result, FailedAttackResult): + continue + elif isinstance(result, SkippedAttackResult): + continue + else: + self.original_candidates.append( + result.original_result.attacked_text + ) + self.successful_candidates.append( + result.perturbed_result.attacked_text + ) + + self.use_obj.model = self.model + self.use_obj.encode = self.encode + print(self.original_candidates) + print(self.successful_candidates) + use_scores = [] + for c in range(len(self.original_candidates)): + use_scores.append(self.use_obj._sim_score(self.original_candidates[c],self.successful_candidates[c])) + + print(use_scores) + + def encode(self, sentences): + return self.model(sentences).numpy() \ No newline at end of file From 0baa5029fc9c3e480c10727acddf8d51e6146d59 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Thu, 2 Sep 2021 19:20:10 -0400 Subject: [PATCH 08/19] [FIX] Working USE --- textattack/metrics/quality_metrics/use.py | 28 ++++++++--------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/textattack/metrics/quality_metrics/use.py b/textattack/metrics/quality_metrics/use.py index c4e3d1ec9..14914e600 100644 --- a/textattack/metrics/quality_metrics/use.py +++ b/textattack/metrics/quality_metrics/use.py @@ -1,9 +1,7 @@ from textattack.attack_results import FailedAttackResult, SkippedAttackResult from textattack.metrics import Metric -from textattack.constraints.semantics.sentence_encoders import SentenceEncoder -from textattack.shared.utils import LazyLoader +from textattack.constraints.semantics.sentence_encoders import UniversalSentenceEncoder -hub = LazyLoader("tensorflow_hub", globals(), "tensorflow_hub") class USEMetric(Metric): @@ -13,18 +11,12 @@ class USEMetric(Metric): def __init__(self, results, **kwargs): self.results = results - self.use_obj = SentenceEncoder() + self.use_obj = UniversalSentenceEncoder() + self.use_obj.model = UniversalSentenceEncoder() self.original_candidates = [] self.successful_candidates = [] + self.all_metrics = {} - if kwargs["large"]: - tfhub_url = "https://tfhub.dev/google/universal-sentence-encoder-large/5" - else: - tfhub_url = "https://tfhub.dev/google/universal-sentence-encoder/4" - - self._tfhub_url = tfhub_url - # Lazily load the model - self.model = hub.load(self._tfhub_url) def calculate(self): for i, result in enumerate(self.results): @@ -40,15 +32,13 @@ def calculate(self): result.perturbed_result.attacked_text ) - self.use_obj.model = self.model - self.use_obj.encode = self.encode - print(self.original_candidates) - print(self.successful_candidates) + use_scores = [] for c in range(len(self.original_candidates)): - use_scores.append(self.use_obj._sim_score(self.original_candidates[c],self.successful_candidates[c])) + use_scores.append(self.use_obj._sim_score(self.original_candidates[c],self.successful_candidates[c]).item()) print(use_scores) - def encode(self, sentences): - return self.model(sentences).numpy() \ No newline at end of file + self.all_metrics['avg_attack_use_score'] = sum(use_scores)/len(use_scores) + + return self.all_metrics \ No newline at end of file From 10ee24b8da2072c4c83c9681a724199229d13816 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Thu, 2 Sep 2021 20:19:42 -0400 Subject: [PATCH 09/19] [FIX] Change structure of Metric mdl --- textattack/loggers/attack_log_manager.py | 17 +++++++++++------ .../metrics/attack_metrics/attack_queries.py | 7 +++---- .../attack_metrics/attack_success_rate.py | 9 +++++---- .../metrics/attack_metrics/words_perturbed.py | 13 +++++++++---- textattack/metrics/metric.py | 4 ++-- .../metrics/quality_metrics/perplexity.py | 11 +++++------ textattack/metrics/quality_metrics/use.py | 9 +++++---- 7 files changed, 40 insertions(+), 30 deletions(-) diff --git a/textattack/loggers/attack_log_manager.py b/textattack/loggers/attack_log_manager.py index 9dcb85315..d2351c386 100644 --- a/textattack/loggers/attack_log_manager.py +++ b/textattack/loggers/attack_log_manager.py @@ -8,7 +8,10 @@ AttackSuccessRate, WordsPerturbed, ) -from textattack.metrics.quality_metrics import Perplexity +from textattack.metrics.quality_metrics import ( + Perplexity, + USEMetric +) from . import CSVLogger, FileLogger, VisdomLogger, WeightsAndBiasesLogger @@ -78,9 +81,9 @@ def log_summary(self): return # Default metrics - calculated on every attack - attack_success_stats = AttackSuccessRate(self.results).calculate() - words_perturbed_stats = WordsPerturbed(self.results).calculate() - attack_query_stats = AttackQueries(self.results).calculate() + attack_success_stats = AttackSuccessRate().calculate(self.results) + words_perturbed_stats = WordsPerturbed().calculate(self.results) + attack_query_stats = AttackQueries().calculate(self.results) # @TODO generate this table based on user input - each column in specific class # Example to demonstrate: @@ -119,7 +122,9 @@ def log_summary(self): ) if self.enable_advance_metrics: - perplexity_stats = Perplexity(self.results).calculate() + perplexity_stats = Perplexity().calculate(self.results) + use_stats = USEMetric(**{"large":False}).calculate(self.results) + print(use_stats) summary_table_rows.append( [ @@ -128,7 +133,7 @@ def log_summary(self): ] ) summary_table_rows.append( - ["Avg Attack Perplexity:", perplexity_stats["avg_attack_perplexity"]] + ["Avg Attack USE Score:", use_stats["avg_attack_use_score"]] ) self.log_summary_rows( diff --git a/textattack/metrics/attack_metrics/attack_queries.py b/textattack/metrics/attack_metrics/attack_queries.py index 5c2658d39..ca7b897ef 100644 --- a/textattack/metrics/attack_metrics/attack_queries.py +++ b/textattack/metrics/attack_metrics/attack_queries.py @@ -12,12 +12,11 @@ class AttackQueries(Metric): Attack results for each instance in dataset """ - def __init__(self, results): - self.results = results - + def __init__(self): self.all_metrics = {} - def calculate(self): + def calculate(self, results): + self.results = results self.num_queries = np.array( [ r.num_queries diff --git a/textattack/metrics/attack_metrics/attack_success_rate.py b/textattack/metrics/attack_metrics/attack_success_rate.py index 6b26b0e0c..b2eccd32f 100644 --- a/textattack/metrics/attack_metrics/attack_success_rate.py +++ b/textattack/metrics/attack_metrics/attack_success_rate.py @@ -10,16 +10,17 @@ class AttackSuccessRate(Metric): Attack results for each instance in dataset """ - def __init__(self, results): - self.results = results + def __init__(self): self.failed_attacks = 0 self.skipped_attacks = 0 self.successful_attacks = 0 - self.total_attacks = len(self.results) self.all_metrics = {} - def calculate(self): + def calculate(self, results): + self.results = results + self.total_attacks = len(self.results) + for i, result in enumerate(self.results): if isinstance(result, FailedAttackResult): self.failed_attacks += 1 diff --git a/textattack/metrics/attack_metrics/words_perturbed.py b/textattack/metrics/attack_metrics/words_perturbed.py index 9558270d3..2ccebd4af 100644 --- a/textattack/metrics/attack_metrics/words_perturbed.py +++ b/textattack/metrics/attack_metrics/words_perturbed.py @@ -5,16 +5,21 @@ class WordsPerturbed(Metric): - def __init__(self, results): + def __init__(self): + self.total_attacks = 0 + self.all_num_words = None + self.perturbed_word_percentages = None + self.num_words_changed_until_success = 0 + self.all_metrics = {} + + def calculate(self, results): self.results = results self.total_attacks = len(self.results) self.all_num_words = np.zeros(len(self.results)) self.perturbed_word_percentages = np.zeros(len(self.results)) self.num_words_changed_until_success = np.zeros(2 ** 16) - self.all_metrics = {} - - def calculate(self): self.max_words_changed = 0 + for i, result in enumerate(self.results): self.all_num_words[i] = len(result.original_result.attacked_text.words) diff --git a/textattack/metrics/metric.py b/textattack/metrics/metric.py index d3a755ac6..11c17c07e 100644 --- a/textattack/metrics/metric.py +++ b/textattack/metrics/metric.py @@ -11,13 +11,13 @@ class Metric(ABC): """A metric for evaluating Adversarial Attack candidates.""" @abstractmethod - def __init__(self, results, **kwargs): + def __init__(self, **kwargs): """Creates pre-built :class:`~textattack.AttackMetric` that correspond to evaluation metrics for adversarial examples. """ raise NotImplementedError() @abstractmethod - def calculate(): + def calculate(self, results): """ Abstract function for computing any values which are to be calculated as a whole during initialization""" raise NotImplementedError diff --git a/textattack/metrics/quality_metrics/perplexity.py b/textattack/metrics/quality_metrics/perplexity.py index 41b3f757b..95ff866c8 100644 --- a/textattack/metrics/quality_metrics/perplexity.py +++ b/textattack/metrics/quality_metrics/perplexity.py @@ -17,8 +17,7 @@ class Perplexity(Metric): Attack results for each instance in dataset """ - def __init__(self, results): - self.results = results + def __init__(self): self.all_metrics = {} self.original_candidates = [] self.successful_candidates = [] @@ -29,11 +28,11 @@ def __init__(self, results): self.max_length = self.ppl_model.config.n_positions self.stride = 128 + def calculate(self, results): + self.results = results self.original_candidates_ppl = [] self.successful_candidates_ppl = [] - def calculate(self): - for i, result in enumerate(self.results): if isinstance(result, FailedAttackResult): continue @@ -74,11 +73,11 @@ def calc_ppl(self, texts): text, add_special_tokens=True ) ).unsqueeze(0) - + # Strided perplexity calculation from huggingface.co/transformers/perplexity.html for i in range(0, input_ids.size(1), self.stride): begin_loc = max(i + self.stride - self.max_length, 0) end_loc = min(i + self.stride, input_ids.size(1)) - trg_len = end_loc - i # may be different from stride on last loop + trg_len = end_loc - i input_ids_t = input_ids[:,begin_loc:end_loc].to(textattack.shared.utils.device) target_ids = input_ids_t.clone() target_ids[:,:-trg_len] = -100 diff --git a/textattack/metrics/quality_metrics/use.py b/textattack/metrics/quality_metrics/use.py index 14914e600..2dfd36fa5 100644 --- a/textattack/metrics/quality_metrics/use.py +++ b/textattack/metrics/quality_metrics/use.py @@ -9,8 +9,7 @@ class USEMetric(Metric): where the text embeddings are created using the Universal Sentence Encoder.""" - def __init__(self, results, **kwargs): - self.results = results + def __init__(self, **kwargs): self.use_obj = UniversalSentenceEncoder() self.use_obj.model = UniversalSentenceEncoder() self.original_candidates = [] @@ -18,7 +17,9 @@ def __init__(self, results, **kwargs): self.all_metrics = {} - def calculate(self): + def calculate(self, results): + self.results = results + for i, result in enumerate(self.results): if isinstance(result, FailedAttackResult): continue @@ -39,6 +40,6 @@ def calculate(self): print(use_scores) - self.all_metrics['avg_attack_use_score'] = sum(use_scores)/len(use_scores) + self.all_metrics['avg_attack_use_score'] = round(sum(use_scores)/len(use_scores),2) return self.all_metrics \ No newline at end of file From 32c3e43adcda3f9b555090f5c060eff2ba1cd5b0 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 10 Sep 2021 02:33:50 -0400 Subject: [PATCH 10/19] [CODE] Fix metrics, add tests --- ...n_attack_hotflip_lstm_mr_4_adv_metrics.txt | 74 +++++++++++++++++++ ...tack_transformers_datasets_adv_metrics.txt | 68 +++++++++++++++++ tests/test_command_line/test_attack.py | 25 +++++++ textattack/loggers/attack_log_manager.py | 19 +++-- textattack/metrics/__init__.py | 2 +- .../attack_metrics/attack_success_rate.py | 2 +- .../metrics/quality_metrics/perplexity.py | 28 +++---- textattack/metrics/quality_metrics/use.py | 36 ++++----- 8 files changed, 210 insertions(+), 44 deletions(-) create mode 100644 tests/sample_outputs/run_attack_hotflip_lstm_mr_4_adv_metrics.txt create mode 100644 tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt diff --git a/tests/sample_outputs/run_attack_hotflip_lstm_mr_4_adv_metrics.txt b/tests/sample_outputs/run_attack_hotflip_lstm_mr_4_adv_metrics.txt new file mode 100644 index 000000000..9023d194b --- /dev/null +++ b/tests/sample_outputs/run_attack_hotflip_lstm_mr_4_adv_metrics.txt @@ -0,0 +1,74 @@ +/.*/Attack( + (search_method): BeamSearch( + (beam_width): 10 + ) + (goal_function): UntargetedClassification + (transformation): WordSwapGradientBased( + (top_n): 1 + ) + (constraints): + (0): MaxWordsPerturbed( + (max_num_words): 2 + (compare_against_original): True + ) + (1): WordEmbeddingDistance( + (embedding): WordEmbedding + (min_cos_sim): 0.8 + (cased): False + (include_unknown_words): True + (compare_against_original): True + ) + (2): PartOfSpeech( + (tagger_type): nltk + (tagset): universal + (allow_verb_noun_swap): True + (compare_against_original): True + ) + (3): RepeatModification + (4): StopwordModification + (is_black_box): False +) + +--------------------------------------------- Result 1 --------------------------------------------- +[[Positive (96%)]] --> [[Negative (77%)]] + +the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur [[supplies]] with tremendous skill . + +the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur [[stagnated]] with tremendous skill . + + +--------------------------------------------- Result 2 --------------------------------------------- +[[Negative (57%)]] --> [[[SKIPPED]]] + +red dragon " never cuts corners . + + +--------------------------------------------- Result 3 --------------------------------------------- +[[Positive (51%)]] --> [[[FAILED]]] + +fresnadillo has something serious to say about the ways in which extravagant chance can distort our perspective and throw us off the path of good sense . + + +--------------------------------------------- Result 4 --------------------------------------------- +[[Positive (89%)]] --> [[[FAILED]]] + +throws in enough clever and unexpected twists to make the formula feel fresh . + + + ++-------------------------------+--------+ +| Attack Results | | ++-------------------------------+--------+ +| Number of successful attacks: | 1 | +| Number of failed attacks: | 2 | +| Number of skipped attacks: | 1 | +| Original accuracy: | 75.0% | +| Accuracy under attack: | 50.0% | +| Attack success rate: | 33.33% | +| Average perturbed word %: | 5.56% | +| Average num. words per input: | 15.5 | +| Avg num queries: | 1.33 | +| Average Original Perplexity: | 291.47 | +| Average Attack Perplexity: | 320.33 | +| Average Attack USE Score: | 0.91 | ++-------------------------------+--------+ diff --git a/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt b/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt new file mode 100644 index 000000000..de158bb32 --- /dev/null +++ b/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt @@ -0,0 +1,68 @@ +/.*/Attack( + (search_method): GreedyWordSwapWIR( + (wir_method): unk + ) + (goal_function): UntargetedClassification + (transformation): CompositeTransformation( + (0): WordSwapNeighboringCharacterSwap( + (random_one): True + ) + (1): WordSwapRandomCharacterSubstitution( + (random_one): True + ) + (2): WordSwapRandomCharacterDeletion( + (random_one): True + ) + (3): WordSwapRandomCharacterInsertion( + (random_one): True + ) + ) + (constraints): + (0): LevenshteinEditDistance( + (max_edit_distance): 30 + (compare_against_original): True + ) + (1): RepeatModification + (2): StopwordModification + (is_black_box): True +) + +--------------------------------------------- Result 1 --------------------------------------------- +[[Negative (100%)]] --> [[Positive (71%)]] + +[[hide]] [[new]] secretions from the parental units + +[[Ehide]] [[enw]] secretions from the parental units + + +--------------------------------------------- Result 2 --------------------------------------------- +[[Negative (100%)]] --> [[[FAILED]]] + +contains no wit , only labored gags + + +--------------------------------------------- Result 3 --------------------------------------------- +[[Positive (100%)]] --> [[Negative (96%)]] + +that [[loves]] its characters and communicates [[something]] [[rather]] [[beautiful]] about human nature + +that [[lodes]] its characters and communicates [[somethNng]] [[rathrer]] [[beautifdul]] about human nature + + + ++-------------------------------+---------+ +| Attack Results | | ++-------------------------------+---------+ +| Number of successful attacks: | 2 | +| Number of failed attacks: | 1 | +| Number of skipped attacks: | 0 | +| Original accuracy: | 100.0% | +| Accuracy under attack: | 33.33% | +| Attack success rate: | 66.67% | +| Average perturbed word %: | 30.95% | +| Average num. words per input: | 8.33 | +| Avg num queries: | 22.67 | +| Average Original Perplexity: | 1126.57 | +| Average Attack Perplexity: | 2823.86 | +| Average Attack USE Score: | 0.76 | ++-------------------------------+---------+ diff --git a/tests/test_command_line/test_attack.py b/tests/test_command_line/test_attack.py index 714f73ed0..f205642fa 100644 --- a/tests/test_command_line/test_attack.py +++ b/tests/test_command_line/test_attack.py @@ -48,6 +48,20 @@ "tests/sample_outputs/run_attack_transformers_datasets.txt", ), # + # test loading an attack from the transformers model hub and calculate perplexity and use + # + ( + "attack_from_transformers", + ( + "textattack attack --model-from-huggingface " + "distilbert-base-uncased-finetuned-sst-2-english " + "--dataset-from-huggingface glue^sst2^train --recipe deepwordbug --num-examples 3 " + "--enable-advance-metrics" + "" + ), + "tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt", + ), + # # test running an attack by loading a model and dataset from file # ( @@ -72,6 +86,17 @@ "tests/sample_outputs/run_attack_hotflip_lstm_mr_4.txt", ), # + # test hotflip on 10 samples from LSTM MR and calculate perplexity and use + # + ( + "run_attack_hotflip_lstm_mr_4", + ( + "textattack attack --model lstm-mr --recipe hotflip " + "--num-examples 4 --num-examples-offset 3 --enable-advance-metrics " + ), + "tests/sample_outputs/run_attack_hotflip_lstm_mr_4_adv_metrics.txt", + ), + # # test: run_attack deepwordbug attack on 10 samples from LSTM MR # ( diff --git a/textattack/loggers/attack_log_manager.py b/textattack/loggers/attack_log_manager.py index d2351c386..e07f1ec9a 100644 --- a/textattack/loggers/attack_log_manager.py +++ b/textattack/loggers/attack_log_manager.py @@ -8,10 +8,7 @@ AttackSuccessRate, WordsPerturbed, ) -from textattack.metrics.quality_metrics import ( - Perplexity, - USEMetric -) +from textattack.metrics.quality_metrics import Perplexity, USEMetric from . import CSVLogger, FileLogger, VisdomLogger, WeightsAndBiasesLogger @@ -123,17 +120,23 @@ def log_summary(self): if self.enable_advance_metrics: perplexity_stats = Perplexity().calculate(self.results) - use_stats = USEMetric(**{"large":False}).calculate(self.results) - print(use_stats) + use_stats = USEMetric().calculate(self.results) summary_table_rows.append( [ - "Avg Original Perplexity:", + "Average Original Perplexity:", perplexity_stats["avg_original_perplexity"], ] ) + + summary_table_rows.append( + [ + "Average Attack Perplexity:", + perplexity_stats["avg_attack_perplexity"], + ] + ) summary_table_rows.append( - ["Avg Attack USE Score:", use_stats["avg_attack_use_score"]] + ["Average Attack USE Score:", use_stats["avg_attack_use_score"]] ) self.log_summary_rows( diff --git a/textattack/metrics/__init__.py b/textattack/metrics/__init__.py index 7bb7bd0bf..fde2faf63 100644 --- a/textattack/metrics/__init__.py +++ b/textattack/metrics/__init__.py @@ -8,4 +8,4 @@ from .attack_metrics import AttackQueries from .quality_metrics import Perplexity -from .quality_metrics import USEMetric \ No newline at end of file +from .quality_metrics import USEMetric diff --git a/textattack/metrics/attack_metrics/attack_success_rate.py b/textattack/metrics/attack_metrics/attack_success_rate.py index b2eccd32f..424ab44ca 100644 --- a/textattack/metrics/attack_metrics/attack_success_rate.py +++ b/textattack/metrics/attack_metrics/attack_success_rate.py @@ -20,7 +20,7 @@ def __init__(self): def calculate(self, results): self.results = results self.total_attacks = len(self.results) - + for i, result in enumerate(self.results): if isinstance(result, FailedAttackResult): self.failed_attacks += 1 diff --git a/textattack/metrics/quality_metrics/perplexity.py b/textattack/metrics/quality_metrics/perplexity.py index 95ff866c8..564bdb158 100644 --- a/textattack/metrics/quality_metrics/perplexity.py +++ b/textattack/metrics/quality_metrics/perplexity.py @@ -49,46 +49,42 @@ def calculate(self, results): ppl_orig = self.calc_ppl(self.original_candidates) ppl_attack = self.calc_ppl(self.successful_candidates) - self.all_metrics["avg_original_perplexity"] = round( - ppl_orig[0], 2 - ) + self.all_metrics["avg_original_perplexity"] = round(ppl_orig[0], 2) self.all_metrics["original_perplexity_list"] = ppl_orig[1] - self.all_metrics["avg_attack_perplexity"] = round( - ppl_attack[0], 2 - ) + self.all_metrics["avg_attack_perplexity"] = round(ppl_attack[0], 2) self.all_metrics["attack_perplexity_list"] = ppl_attack[1] return self.all_metrics def calc_ppl(self, texts): - + ppl_vals = [] with torch.no_grad(): for text in texts: eval_loss = [] input_ids = torch.tensor( - self.ppl_tokenizer.encode( - text, add_special_tokens=True - ) + self.ppl_tokenizer.encode(text, add_special_tokens=True) ).unsqueeze(0) # Strided perplexity calculation from huggingface.co/transformers/perplexity.html for i in range(0, input_ids.size(1), self.stride): begin_loc = max(i + self.stride - self.max_length, 0) end_loc = min(i + self.stride, input_ids.size(1)) trg_len = end_loc - i - input_ids_t = input_ids[:,begin_loc:end_loc].to(textattack.shared.utils.device) + input_ids_t = input_ids[:, begin_loc:end_loc].to( + textattack.shared.utils.device + ) target_ids = input_ids_t.clone() - target_ids[:,:-trg_len] = -100 + target_ids[:, :-trg_len] = -100 outputs = self.ppl_model(input_ids_t, labels=target_ids) log_likelihood = outputs[0] * trg_len eval_loss.append(log_likelihood) - ppl_vals.append(torch.exp(torch.stack(eval_loss).sum() / end_loc).item()) - - - return sum(ppl_vals)/len(ppl_vals), ppl_vals + ppl_vals.append( + torch.exp(torch.stack(eval_loss).sum() / end_loc).item() + ) + return sum(ppl_vals) / len(ppl_vals), ppl_vals diff --git a/textattack/metrics/quality_metrics/use.py b/textattack/metrics/quality_metrics/use.py index 2dfd36fa5..989104fc4 100644 --- a/textattack/metrics/quality_metrics/use.py +++ b/textattack/metrics/quality_metrics/use.py @@ -3,11 +3,13 @@ from textattack.constraints.semantics.sentence_encoders import UniversalSentenceEncoder - class USEMetric(Metric): - """Constraint using similarity between sentence encodings of x and x_adv - where the text embeddings are created using the Universal Sentence - Encoder.""" + """Calculates average USE similarity on all successfull attacks + + Args: + results (:obj::`list`:class:`~textattack.goal_function_results.GoalFunctionResult`): + Attack results for each instance in dataset + """ def __init__(self, **kwargs): self.use_obj = UniversalSentenceEncoder() @@ -16,7 +18,6 @@ def __init__(self, **kwargs): self.successful_candidates = [] self.all_metrics = {} - def calculate(self, results): self.results = results @@ -26,20 +27,19 @@ def calculate(self, results): elif isinstance(result, SkippedAttackResult): continue else: - self.original_candidates.append( - result.original_result.attacked_text - ) - self.successful_candidates.append( - result.perturbed_result.attacked_text - ) - - + self.original_candidates.append(result.original_result.attacked_text) + self.successful_candidates.append(result.perturbed_result.attacked_text) + use_scores = [] for c in range(len(self.original_candidates)): - use_scores.append(self.use_obj._sim_score(self.original_candidates[c],self.successful_candidates[c]).item()) - - print(use_scores) + use_scores.append( + self.use_obj._sim_score( + self.original_candidates[c], self.successful_candidates[c] + ).item() + ) - self.all_metrics['avg_attack_use_score'] = round(sum(use_scores)/len(use_scores),2) + self.all_metrics["avg_attack_use_score"] = round( + sum(use_scores) / len(use_scores), 2 + ) - return self.all_metrics \ No newline at end of file + return self.all_metrics From 559057e068de0ee35116427e871af48f4376c0fb Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 10 Sep 2021 02:38:55 -0400 Subject: [PATCH 11/19] [CODE] Fix black on use --- textattack/metrics/quality_metrics/use.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/textattack/metrics/quality_metrics/use.py b/textattack/metrics/quality_metrics/use.py index 989104fc4..cd791a3a6 100644 --- a/textattack/metrics/quality_metrics/use.py +++ b/textattack/metrics/quality_metrics/use.py @@ -1,6 +1,6 @@ from textattack.attack_results import FailedAttackResult, SkippedAttackResult -from textattack.metrics import Metric from textattack.constraints.semantics.sentence_encoders import UniversalSentenceEncoder +from textattack.metrics import Metric class USEMetric(Metric): From aab7eec818d94548d04ded0d629aa91b5cab4926 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 10 Sep 2021 02:45:35 -0400 Subject: [PATCH 12/19] [CODE] Fix isort on use --- textattack/metrics/quality_metrics/perplexity.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/textattack/metrics/quality_metrics/perplexity.py b/textattack/metrics/quality_metrics/perplexity.py index 564bdb158..c6aeb693b 100644 --- a/textattack/metrics/quality_metrics/perplexity.py +++ b/textattack/metrics/quality_metrics/perplexity.py @@ -1,8 +1,5 @@ -import re - import torch -import tqdm -from transformers import AutoTokenizer, GPT2LMHeadModel, GPT2Tokenizer +from transformers import GPT2LMHeadModel, GPT2Tokenizer from textattack.attack_results import FailedAttackResult, SkippedAttackResult from textattack.metrics import Metric From b5a120987f939cd45e9d47a76150c1a32227bb07 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 10 Sep 2021 02:54:38 -0400 Subject: [PATCH 13/19] [CODE] Add new help msg --- textattack/attack_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/textattack/attack_args.py b/textattack/attack_args.py index 24f745f32..ed6808c58 100644 --- a/textattack/attack_args.py +++ b/textattack/attack_args.py @@ -357,7 +357,7 @@ def _add_parser_args(cls, parser): "--enable-advance-metrics", action="store_true", default=default_obj.enable_advance_metrics, - help="Enable advance metric calculations", + help="Enable calculation and display of optional advance post-hoc metrics like perplexity, use distance, etc.", ) return parser From aa1ad1579896ea095e750414ce95a271604bc849 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 10 Sep 2021 10:46:45 -0400 Subject: [PATCH 14/19] [CODE] Add new docs --- tests/test_command_line/test_attack.py | 6 +++--- textattack/metrics/metric.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_command_line/test_attack.py b/tests/test_command_line/test_attack.py index f205642fa..1bd5c3a27 100644 --- a/tests/test_command_line/test_attack.py +++ b/tests/test_command_line/test_attack.py @@ -51,7 +51,7 @@ # test loading an attack from the transformers model hub and calculate perplexity and use # ( - "attack_from_transformers", + "attack_from_transformers_adv_metrics", ( "textattack attack --model-from-huggingface " "distilbert-base-uncased-finetuned-sst-2-english " @@ -89,7 +89,7 @@ # test hotflip on 10 samples from LSTM MR and calculate perplexity and use # ( - "run_attack_hotflip_lstm_mr_4", + "run_attack_hotflip_lstm_mr_4_adv_metrics", ( "textattack attack --model lstm-mr --recipe hotflip " "--num-examples 4 --num-examples-offset 3 --enable-advance-metrics " @@ -192,7 +192,6 @@ ), ] - @pytest.mark.parametrize("name, command, sample_output_file", attack_test_params) @pytest.mark.slow def test_command_line_attack(name, command, sample_output_file): @@ -221,6 +220,7 @@ def test_command_line_attack(name, command, sample_output_file): if DEBUG and not re.match(desired_re, stdout, flags=re.S): pdb.set_trace() + print(re.match(desired_re, stdout, flags=re.S)) assert re.match(desired_re, stdout, flags=re.S) assert result.returncode == 0, "return code not 0" diff --git a/textattack/metrics/metric.py b/textattack/metrics/metric.py index 11c17c07e..8861b064b 100644 --- a/textattack/metrics/metric.py +++ b/textattack/metrics/metric.py @@ -12,7 +12,7 @@ class Metric(ABC): @abstractmethod def __init__(self, **kwargs): - """Creates pre-built :class:`~textattack.AttackMetric` that correspond to + """Creates pre-built :class:`~textattack.Metric` that correspond to evaluation metrics for adversarial examples. """ raise NotImplementedError() From d44a54c787f5117c575b5a52e767c77201d3a1f5 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 10 Sep 2021 10:48:04 -0400 Subject: [PATCH 15/19] [CODE] Fix print --- tests/test_command_line/test_attack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_command_line/test_attack.py b/tests/test_command_line/test_attack.py index 1bd5c3a27..5458dd8ea 100644 --- a/tests/test_command_line/test_attack.py +++ b/tests/test_command_line/test_attack.py @@ -220,7 +220,6 @@ def test_command_line_attack(name, command, sample_output_file): if DEBUG and not re.match(desired_re, stdout, flags=re.S): pdb.set_trace() - print(re.match(desired_re, stdout, flags=re.S)) assert re.match(desired_re, stdout, flags=re.S) assert result.returncode == 0, "return code not 0" From eab1cd06d37c7d8cc78fee9a2bde635646cd38d3 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Fri, 10 Sep 2021 10:52:54 -0400 Subject: [PATCH 16/19] [CODE] Fix black --- tests/test_command_line/test_attack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_command_line/test_attack.py b/tests/test_command_line/test_attack.py index 5458dd8ea..5a455d22d 100644 --- a/tests/test_command_line/test_attack.py +++ b/tests/test_command_line/test_attack.py @@ -192,6 +192,7 @@ ), ] + @pytest.mark.parametrize("name, command, sample_output_file", attack_test_params) @pytest.mark.slow def test_command_line_attack(name, command, sample_output_file): From 3e2b16f344f68d586dac702c9ae5bb7da57dcbd4 Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Thu, 16 Sep 2021 18:05:01 -0400 Subject: [PATCH 17/19] [FIX] Fix perplexity precision --- .../run_attack_transformers_datasets_adv_metrics.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt b/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt index de158bb32..ba802b05a 100644 --- a/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt +++ b/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt @@ -63,6 +63,6 @@ that [[lodes]] its characters and communicates [[somethNng]] [[rathrer]] [[beaut | Average num. words per input: | 8.33 | | Avg num queries: | 22.67 | | Average Original Perplexity: | 1126.57 | -| Average Attack Perplexity: | 2823.86 | +| Average Attack Perplexity: | 2823.88 | | Average Attack USE Score: | 0.76 | +-------------------------------+---------+ From f1ef471ea8878038066e7918a93c301eb915832f Mon Sep 17 00:00:00 2001 From: sanchit97 Date: Thu, 16 Sep 2021 19:53:16 -0400 Subject: [PATCH 18/19] [FIX] Fix perplexity escape seq --- .../run_attack_transformers_datasets_adv_metrics.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt b/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt index ba802b05a..1b01102f7 100644 --- a/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt +++ b/tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt @@ -63,6 +63,6 @@ that [[lodes]] its characters and communicates [[somethNng]] [[rathrer]] [[beaut | Average num. words per input: | 8.33 | | Avg num queries: | 22.67 | | Average Original Perplexity: | 1126.57 | -| Average Attack Perplexity: | 2823.88 | +| Average Attack Perplexity: | 2823/.*/| | Average Attack USE Score: | 0.76 | +-------------------------------+---------+ From fa9817af5dccd690655e226f8749d51e01fb89e1 Mon Sep 17 00:00:00 2001 From: Yanjun Qi Date: Wed, 29 Sep 2021 15:50:57 -0400 Subject: [PATCH 19/19] fix docstring issues.. --- ...traints.grammaticality.language_models.rst | 1 + .../textattack.constraints.grammaticality.rst | 1 + docs/apidoc/textattack.constraints.rst | 1 + .../textattack.constraints.semantics.rst | 1 + ...onstraints.semantics.sentence_encoders.rst | 1 + docs/apidoc/textattack.datasets.rst | 1 + docs/apidoc/textattack.goal_functions.rst | 1 + .../textattack.metrics.attack_metrics.rst | 26 +++++++++++++++++++ .../textattack.metrics.quality_metrics.rst | 20 ++++++++++++++ docs/apidoc/textattack.metrics.rst | 22 ++++++++++++++++ docs/apidoc/textattack.models.rst | 1 + docs/apidoc/textattack.rst | 1 + docs/apidoc/textattack.shared.rst | 1 + docs/apidoc/textattack.transformations.rst | 1 + textattack/attack_args.py | 2 +- textattack/metrics/attack_metrics/__init__.py | 3 +-- .../metrics/attack_metrics/attack_queries.py | 21 ++++++++++----- .../attack_metrics/attack_success_rate.py | 20 +++++++++----- .../metrics/attack_metrics/words_perturbed.py | 14 ++++++++++ textattack/metrics/metric.py | 7 ++++- .../metrics/quality_metrics/__init__.py | 4 +-- .../metrics/quality_metrics/perplexity.py | 22 ++++++++++------ textattack/metrics/quality_metrics/use.py | 13 +++++----- 23 files changed, 150 insertions(+), 35 deletions(-) create mode 100644 docs/apidoc/textattack.metrics.attack_metrics.rst create mode 100644 docs/apidoc/textattack.metrics.quality_metrics.rst create mode 100644 docs/apidoc/textattack.metrics.rst diff --git a/docs/apidoc/textattack.constraints.grammaticality.language_models.rst b/docs/apidoc/textattack.constraints.grammaticality.language_models.rst index f342ed86e..d998a19d9 100644 --- a/docs/apidoc/textattack.constraints.grammaticality.language_models.rst +++ b/docs/apidoc/textattack.constraints.grammaticality.language_models.rst @@ -7,6 +7,7 @@ textattack.constraints.grammaticality.language\_models package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.constraints.grammaticality.rst b/docs/apidoc/textattack.constraints.grammaticality.rst index e39cdb404..f3d2c34c0 100644 --- a/docs/apidoc/textattack.constraints.grammaticality.rst +++ b/docs/apidoc/textattack.constraints.grammaticality.rst @@ -7,6 +7,7 @@ textattack.constraints.grammaticality package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.constraints.rst b/docs/apidoc/textattack.constraints.rst index 72dbb9e46..1907e29b4 100644 --- a/docs/apidoc/textattack.constraints.rst +++ b/docs/apidoc/textattack.constraints.rst @@ -7,6 +7,7 @@ textattack.constraints package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.constraints.semantics.rst b/docs/apidoc/textattack.constraints.semantics.rst index 3e8b0973f..e20d9c0e1 100644 --- a/docs/apidoc/textattack.constraints.semantics.rst +++ b/docs/apidoc/textattack.constraints.semantics.rst @@ -7,6 +7,7 @@ textattack.constraints.semantics package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.constraints.semantics.sentence_encoders.rst b/docs/apidoc/textattack.constraints.semantics.sentence_encoders.rst index 9e712dd86..22be1b971 100644 --- a/docs/apidoc/textattack.constraints.semantics.sentence_encoders.rst +++ b/docs/apidoc/textattack.constraints.semantics.sentence_encoders.rst @@ -7,6 +7,7 @@ textattack.constraints.semantics.sentence\_encoders package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.datasets.rst b/docs/apidoc/textattack.datasets.rst index f4881aa65..d5e1564db 100644 --- a/docs/apidoc/textattack.datasets.rst +++ b/docs/apidoc/textattack.datasets.rst @@ -7,6 +7,7 @@ textattack.datasets package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.goal_functions.rst b/docs/apidoc/textattack.goal_functions.rst index 4e42db9b1..a1a429f80 100644 --- a/docs/apidoc/textattack.goal_functions.rst +++ b/docs/apidoc/textattack.goal_functions.rst @@ -7,6 +7,7 @@ textattack.goal\_functions package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.metrics.attack_metrics.rst b/docs/apidoc/textattack.metrics.attack_metrics.rst new file mode 100644 index 000000000..b6ff602c9 --- /dev/null +++ b/docs/apidoc/textattack.metrics.attack_metrics.rst @@ -0,0 +1,26 @@ +textattack.metrics.attack\_metrics package +========================================== + +.. automodule:: textattack.metrics.attack_metrics + :members: + :undoc-members: + :show-inheritance: + + + +.. automodule:: textattack.metrics.attack_metrics.attack_queries + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: textattack.metrics.attack_metrics.attack_success_rate + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: textattack.metrics.attack_metrics.words_perturbed + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/apidoc/textattack.metrics.quality_metrics.rst b/docs/apidoc/textattack.metrics.quality_metrics.rst new file mode 100644 index 000000000..6d46e32db --- /dev/null +++ b/docs/apidoc/textattack.metrics.quality_metrics.rst @@ -0,0 +1,20 @@ +textattack.metrics.quality\_metrics package +=========================================== + +.. automodule:: textattack.metrics.quality_metrics + :members: + :undoc-members: + :show-inheritance: + + + +.. automodule:: textattack.metrics.quality_metrics.perplexity + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: textattack.metrics.quality_metrics.use + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/apidoc/textattack.metrics.rst b/docs/apidoc/textattack.metrics.rst new file mode 100644 index 000000000..bcad2dbe0 --- /dev/null +++ b/docs/apidoc/textattack.metrics.rst @@ -0,0 +1,22 @@ +textattack.metrics package +========================== + +.. automodule:: textattack.metrics + :members: + :undoc-members: + :show-inheritance: + + + +.. toctree:: + :maxdepth: 6 + + textattack.metrics.attack_metrics + textattack.metrics.quality_metrics + + + +.. automodule:: textattack.metrics.metric + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/apidoc/textattack.models.rst b/docs/apidoc/textattack.models.rst index 84074b073..153747f05 100644 --- a/docs/apidoc/textattack.models.rst +++ b/docs/apidoc/textattack.models.rst @@ -7,6 +7,7 @@ textattack.models package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.rst b/docs/apidoc/textattack.rst index 83ff65b57..325e4deab 100644 --- a/docs/apidoc/textattack.rst +++ b/docs/apidoc/textattack.rst @@ -19,6 +19,7 @@ textattack package textattack.goal_function_results textattack.goal_functions textattack.loggers + textattack.metrics textattack.models textattack.search_methods textattack.shared diff --git a/docs/apidoc/textattack.shared.rst b/docs/apidoc/textattack.shared.rst index 9679c45a7..34a5a1b4d 100644 --- a/docs/apidoc/textattack.shared.rst +++ b/docs/apidoc/textattack.shared.rst @@ -7,6 +7,7 @@ textattack.shared package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/docs/apidoc/textattack.transformations.rst b/docs/apidoc/textattack.transformations.rst index 1ae3f6533..6e87f53fb 100644 --- a/docs/apidoc/textattack.transformations.rst +++ b/docs/apidoc/textattack.transformations.rst @@ -7,6 +7,7 @@ textattack.transformations package :show-inheritance: + .. toctree:: :maxdepth: 6 diff --git a/textattack/attack_args.py b/textattack/attack_args.py index ed6808c58..73dee5502 100644 --- a/textattack/attack_args.py +++ b/textattack/attack_args.py @@ -357,7 +357,7 @@ def _add_parser_args(cls, parser): "--enable-advance-metrics", action="store_true", default=default_obj.enable_advance_metrics, - help="Enable calculation and display of optional advance post-hoc metrics like perplexity, use distance, etc.", + help="Enable calculation and display of optional advance post-hoc metrics like perplexity, USE distance, etc.", ) return parser diff --git a/textattack/metrics/attack_metrics/__init__.py b/textattack/metrics/attack_metrics/__init__.py index 1cce96dcb..3eb90e343 100644 --- a/textattack/metrics/attack_metrics/__init__.py +++ b/textattack/metrics/attack_metrics/__init__.py @@ -3,8 +3,7 @@ attack_metrics: ====================== -TextAttack allows users to use their own metrics on adversarial examples or select common metrics to display. - +TextAttack provide users common metrics on attacks' quality. """ diff --git a/textattack/metrics/attack_metrics/attack_queries.py b/textattack/metrics/attack_metrics/attack_queries.py index ca7b897ef..7affc698c 100644 --- a/textattack/metrics/attack_metrics/attack_queries.py +++ b/textattack/metrics/attack_metrics/attack_queries.py @@ -1,3 +1,10 @@ +""" + +Metrics on AttackQueries +========================= + +""" + import numpy as np from textattack.attack_results import SkippedAttackResult @@ -5,17 +12,17 @@ class AttackQueries(Metric): - """Calculates all metrics related to number of queries in an attack - - Args: - results (:obj::`list`:class:`~textattack.goal_function_results.GoalFunctionResult`): - Attack results for each instance in dataset - """ - def __init__(self): self.all_metrics = {} def calculate(self, results): + """Calculates all metrics related to number of queries in an attack + + Args: + results (``AttackResult`` objects): + Attack results for each instance in dataset + """ + self.results = results self.num_queries = np.array( [ diff --git a/textattack/metrics/attack_metrics/attack_success_rate.py b/textattack/metrics/attack_metrics/attack_success_rate.py index 424ab44ca..368e8b876 100644 --- a/textattack/metrics/attack_metrics/attack_success_rate.py +++ b/textattack/metrics/attack_metrics/attack_success_rate.py @@ -1,15 +1,15 @@ +""" + +Metrics on AttackSuccessRate +============================= + +""" + from textattack.attack_results import FailedAttackResult, SkippedAttackResult from textattack.metrics import Metric class AttackSuccessRate(Metric): - """Calculates all metrics related to number of succesful, failed and skipped results in an attack - - Args: - results (:obj::`list`:class:`~textattack.goal_function_results.GoalFunctionResult`): - Attack results for each instance in dataset - """ - def __init__(self): self.failed_attacks = 0 self.skipped_attacks = 0 @@ -18,6 +18,12 @@ def __init__(self): self.all_metrics = {} def calculate(self, results): + """Calculates all metrics related to number of succesful, failed and skipped results in an attack + + Args: + results (``AttackResult`` objects): + Attack results for each instance in dataset + """ self.results = results self.total_attacks = len(self.results) diff --git a/textattack/metrics/attack_metrics/words_perturbed.py b/textattack/metrics/attack_metrics/words_perturbed.py index 2ccebd4af..a9f29b92c 100644 --- a/textattack/metrics/attack_metrics/words_perturbed.py +++ b/textattack/metrics/attack_metrics/words_perturbed.py @@ -1,3 +1,10 @@ +""" + +Metrics on perturbed words +============================= + +""" + import numpy as np from textattack.attack_results import FailedAttackResult, SkippedAttackResult @@ -13,6 +20,13 @@ def __init__(self): self.all_metrics = {} def calculate(self, results): + """Calculates all metrics related to perturbed words in an attack + + Args: + results (``AttackResult`` objects): + Attack results for each instance in dataset + """ + self.results = results self.total_attacks = len(self.results) self.all_num_words = np.zeros(len(self.results)) diff --git a/textattack/metrics/metric.py b/textattack/metrics/metric.py index 8861b064b..1a7c79c0d 100644 --- a/textattack/metrics/metric.py +++ b/textattack/metrics/metric.py @@ -19,5 +19,10 @@ def __init__(self, **kwargs): @abstractmethod def calculate(self, results): - """ Abstract function for computing any values which are to be calculated as a whole during initialization""" + """Abstract function for computing any values which are to be calculated as a whole during initialization + Args: + results (``AttackResult`` objects): + Attack results for each instance in dataset + """ + raise NotImplementedError diff --git a/textattack/metrics/quality_metrics/__init__.py b/textattack/metrics/quality_metrics/__init__.py index 79f707ffc..5addbad4d 100644 --- a/textattack/metrics/quality_metrics/__init__.py +++ b/textattack/metrics/quality_metrics/__init__.py @@ -1,9 +1,9 @@ """ -perplexity: +Metrics on Quality ====================== -TextAttack allows users to use their own metrics on adversarial examples or select common metrics to display. +TextAttack provide users common metrics on text examples' quality. """ diff --git a/textattack/metrics/quality_metrics/perplexity.py b/textattack/metrics/quality_metrics/perplexity.py index c6aeb693b..d508e29fb 100644 --- a/textattack/metrics/quality_metrics/perplexity.py +++ b/textattack/metrics/quality_metrics/perplexity.py @@ -1,3 +1,10 @@ +""" + +Perplexity Metric: +====================== + +""" + import torch from transformers import GPT2LMHeadModel, GPT2Tokenizer @@ -7,13 +14,6 @@ class Perplexity(Metric): - """Calculates average Perplexity on all successfull attacks using a pre-trained small GPT-2 model - - Args: - results (:obj::`list`:class:`~textattack.goal_function_results.GoalFunctionResult`): - Attack results for each instance in dataset - """ - def __init__(self): self.all_metrics = {} self.original_candidates = [] @@ -23,9 +23,15 @@ def __init__(self): self.ppl_tokenizer = GPT2Tokenizer.from_pretrained("gpt2") self.ppl_model.eval() self.max_length = self.ppl_model.config.n_positions - self.stride = 128 + self.stride = 512 def calculate(self, results): + """Calculates average Perplexity on all successfull attacks using a pre-trained small GPT-2 model + + Args: + results (``AttackResult`` objects): + Attack results for each instance in dataset + """ self.results = results self.original_candidates_ppl = [] self.successful_candidates_ppl = [] diff --git a/textattack/metrics/quality_metrics/use.py b/textattack/metrics/quality_metrics/use.py index cd791a3a6..424727cf2 100644 --- a/textattack/metrics/quality_metrics/use.py +++ b/textattack/metrics/quality_metrics/use.py @@ -4,13 +4,6 @@ class USEMetric(Metric): - """Calculates average USE similarity on all successfull attacks - - Args: - results (:obj::`list`:class:`~textattack.goal_function_results.GoalFunctionResult`): - Attack results for each instance in dataset - """ - def __init__(self, **kwargs): self.use_obj = UniversalSentenceEncoder() self.use_obj.model = UniversalSentenceEncoder() @@ -19,6 +12,12 @@ def __init__(self, **kwargs): self.all_metrics = {} def calculate(self, results): + """Calculates average USE similarity on all successfull attacks + + Args: + results (``AttackResult`` objects): + Attack results for each instance in dataset + """ self.results = results for i, result in enumerate(self.results):