Merge pull request #53 from IndicoDataSolutions/lily/rationalized_clean

Lily/rationalized clean

Author: lhz1029

enso/config_rationalized.py

@@ -0,0 +1,168 @@
+import indicoio
+from enso.mode import ModeKeys
+import multiprocessing
+
+"""Constants to configure the rest of Enso."""
+
+# Directory for storing data
+DATA_DIRECTORY = "Data"
+
+# Directory for storing results
+RESULTS_DIRECTORY = "Results"
+
+# Directory for storing features
+FEATURES_DIRECTORY = "Features"
+
+# Directory for storing experiment results
+EXPERIMENT_NAME = "Rationales"
+
+# Name of the csv used to store results
+RESULTS_CSV_NAME = "Results.csv"
+
+# Datasets to featurize or run experiments on
+DATA = {
+ # "Classify/AirlineComplaints",
+ # "Classify/AirlineNegativity",cRep
+ # "Classify/IMDB",
+ # "Classify/Irony",
+ # "Classify/MPQA",
+ # "Classify/MovieReviews",
+ # "Classify/NewYearsResolutions",
+ # "Classify/PoliticalTweetAlignment",
+ # "Classify/PoliticalTweetBias",
+ # "Classify/PoliticalTweetClassification",
+ # "Classify/PoliticalTweetSubjectivity",
+ # "Classify/PoliticalTweetTarget",
+ # "Classify/ReligiousTexts",
+ # "Classify/ShortAnswer",
+ # "Classify/SocialMediaDisasters",
+ # "Classify/Subjectivity",
+ # "Classify/TextSpam",
+ # "Classify/SST-binary"
+ # Seqence
+ # 'SequenceLabeling/Reuters-128',
+ # "SequenceLabeling/table_synth",
+ # 'SequenceLabeling/bonds_new',
+ # 'SequenceLabeling/tables',
+ # 'SequenceLabeling/typed_cols',
+ # 'SequenceLabeling/brown_all',
+ # 'SequenceLabeling/brown_nouns',
+ # 'SequenceLabeling/brown_verbs',
+ # 'SequenceLabeling/brown_pronouns',
+ # 'SequenceLabeling/brown_adverbs',
+ # 'RationalizedClassify/short_bank_qualified',
+ # 'RationalizedClassify/bank_qualified',
+ # 'RationalizedClassify/evidence_inference',
+ # 'RationalizedClassify/federal_tax',
+ # "RationalizedClassify/short_federal_tax",
+ # 'RationalizedClassify/interest_frequency',
+ # "RationalizedClassify/short_interest_frequency",
+ "RationalizedClassify/aviation",
+ # "RationalizedClassify/movie_reviews",
+ # "RationalizedClassify/mining_rationales",
+ # "RationalizedClassify/mining_extractions",
+ # "RationalizedClassify/insurance_rationales",
+ # "RationalizedClassify/insurance_extractions",
+ # "RationalizedClassify/mining",
+ # "RationalizedClassify/insurance_rationales_precise",
+ # 'RationalizedClassify/short_bank_qualified',
+ # 'RationalizedClassify/bank_qualified',
+ # 'RationalizedClassify/short_bank_qualified_fixed',
+ # 'RationalizedClassify/bank_qualified_fixed',
+ # 'RationalizedClassify/short_bank_qualified_precise',
+ # 'RationalizedClassify/bank_qualified_precise',
+}
+
+# Featurizers to activate
+FEATURIZERS = {
+ "PlainTextFeaturizer",
+ # "TextContextFeaturizer",
+ # "IndicoStandard",
+ "SpacyGloveFeaturizer",
+ # "IndicoFastText",
+ # "IndicoSentiment",
+ # "IndicoElmo",
+ # "IndicoTopics",
+ # "IndicoFinance",
+ # "IndicoTransformer",
+ # "IndicoEmotion",
+ # "IndicoFastText",
+ # "SpacyCNNFeaturizer",
+}
+
+# Experiments to run
+EXPERIMENTS = {
+ # "FinetuneSequenceLabel",
+ # "Proto",
+ # "IndicoSequenceLabel"
+ "LRBaselineNonRationalized",
+ "DistReweightedGloveClassifierCV",
+ 'DistReweightedGloveByClassClassifierCV'
+ # "RationaleInformedLRCV"
+ # "FinetuneSeqBaselineRationalized",
+ # "FinetuneClfBaselineNonRationalized",
+ # "LogisticRegressionCV",
+ # "KNNCV",
+ # "TfidfKNN",
+ # "TfidfLogisticRegression",
+ # "KCenters",
+ # "TfidfKCenters"
+ # "SupportVectorMachineCV",
+}
+
+# Metrics to compute
+METRICS = {
+ # "Accuracy",
+ "AccuracyRationalized",
+ "MacroRocAucRationalized",
+ # "MacroRocAuc",
+ # "MacroCharF1",
+ # "MacroCharRecall",
+ # "MacroCharPrecision",
+}
+
+# Test setup metadata
+TEST_SETUP = {
+ "train_sizes": [20, 40, 60, 80, 100, 150, 200, 300, 400, 500],
+ "n_splits": 5,
+ # "samplers": ['RandomRationalized'],
+ # "samplers": ["ImbalanceSampler"],
+ "samplers": ["RandomRationalized"],
+ "sampling_size": 0.2,
+ "resamplers": ["NoResampler"]
+ # "resamplers": ["RandomOverSampler"],
+}
+
+# Visualizations to display
+VISUALIZATIONS = {"FacetGridVisualizer"}
+
+# kwargs to pass directly into visualizations
+VISUALIZATION_OPTIONS = {
+ "display": True,
+ "save": True,
+ "FacetGridVisualizer": {
+ "x_tile": "Metric",
+ "y_tile": "Dataset",
+ "x_axis": "TrainSize",
+ "y_axis": "Result",
+ "lines": ["Experiment", "Featurizer", "Sampler", "Resampler"],
+ "category": "merge",
+ "cv": "mean",
+ "filename": "TestResult",
+ },
+}
+
+MODE = ModeKeys.RATIONALIZED
+
+N_GPUS = 0
+N_CORES = 1 # multiprocessing.cpu_count()
+
+FIX_REQUIREMENTS = True
+
+GOLD_FRAC = 0.05
+CORRUPTION_FRAC = 0.4
+
+indicoio.config.api_key = ""
+
+# If we have no experiment hyperparameters we hope to modify:
+EXPERIMENT_PARAMS = {}

enso/experiment/rationalized.py

@@ -9,7 +9,7 @@
 from enso.experiment.grid_search import GridSearch
 from finetune import Classifier, SequenceLabeler
 from sklearn.preprocessing import LabelBinarizer
-from collections import Counter, defaultdict
+from collections import Counter, defaultdict, OrderedDict
 
 class RationalizedGridSearch(GridSearch):
  def fit(self, X, y):
@@ -101,7 +101,10 @@ def cleanup(self):
 
 @Registry.register_experiment(ModeKeys.RATIONALIZED, requirements=[("Featurizer", "PlainTextFeaturizer")])
 class ReweightedGloveClassifier(ClassificationExperiment):
+ """
+ Weights words by their proportional occurrence as rationales, smoothed
 
+ """
  NLP = None
 
  def __init__(self, *args, **kwargs):
@@ -211,14 +214,10 @@ def fit(self, X, Y):
  rationales.append([{**label, "label": l[1]} for label in l[0]])
  else:
  rationales.append([])
- rationale_texts = [
- rationale['text'] 
- for doc in rationales 
- for rationale in doc
- ]
- docs = np.asarray([self.NLP(str(x), disable=['ner', 'tagger', 'textcat']) for x in X])
- rationale_docs = np.asarray([self.NLP(rationale) for rationale in rationale_texts if len(rationale)])
- self._train_rationale_model(docs, rationale_docs)
+ rationale_texts = [rationale["text"] for doc in rationales for rationale in doc]
+ docs = self.NLP.pipe(X, disable=["ner", "tagger", "textcat"])
+ rationale_docs = np.asarray([self.NLP(rationale) if len(rationale) else None for rationale in rationale_texts])
+ self._train_rationale_model(docs, rationale_docs, labels=labels)
 
  doc_vects = np.asarray([self._featurize(doc) for doc in docs])
  resampled_x, resampled_y = self.resample(doc_vects, labels)
@@ -236,12 +235,15 @@ def predict(self, X, **kwargs):
 
 @Registry.register_experiment(ModeKeys.RATIONALIZED, requirements=[("Featurizer", "PlainTextFeaturizer")])
 class DistReweightedGloveClassifierCV(BaseRationaleGridSearch):
+ """
+ Weights words by cosine similarity to the mean of the rationale vector representations
 
- def _train_rationale_model(self, docs, rationale_docs):
+ """
+ def _train_rationale_model(self, docs, rationale_docs, labels=None):
  rationale_vecs = [
- doc.vector / np.linalg.norm(doc.vector) 
- for doc in rationale_docs 
- if doc.has_vector and np.any(np.nonzero(doc.vector))
+ doc.vector / np.linalg.norm(doc.vector)
+ for doc in rationale_docs
+ if doc and doc.has_vector and np.any(np.nonzero(doc.vector))
  ]
  rationale_proto = np.mean(rationale_vecs, axis=0)
  self.normalized_rationale_proto = rationale_proto / np.linalg.norm(rationale_proto)
@@ -264,21 +266,71 @@ def _featurize(self, doc):
 
 
 @Registry.register_experiment(ModeKeys.RATIONALIZED, requirements=[("Featurizer", "PlainTextFeaturizer")])
-class RationaleInformedLRCV(BaseRationaleGridSearch):
+class DistReweightedGloveByClassClassifierCV(BaseRationaleGridSearch):
+ """
+ Weights words by cosine similarity to the mean of the rationale vector representations per class
+
+ """
+ def _train_rationale_model(self, docs, rationale_docs, labels=None):
+ rationale_vecs_by_class = defaultdict(list)
+ for doc, label in zip(rationale_docs, labels):
+ if doc and doc.has_vector and np.any(np.nonzero(doc.vector)):
+ rationale_vecs_by_class[label].append(
+ doc.vector / np.linalg.norm(doc.vector)
+ )
+ rationale_proto_by_class = {
+ label: np.mean(rationale_vecs, axis=0)
+ for label, rationale_vecs in rationale_vecs_by_class.items()
+ }
+ self.normalized_rationale_proto_by_class = OrderedDict({
+ label: rationale_proto / np.linalg.norm(rationale_proto)
+ for label, rationale_proto in rationale_proto_by_class.items()
+ })
 
- def _train_rationale_model(self, docs, rationale_docs):
+ def _rationale_weight(self, word, rationale_proto):
+ cosine_sim = np.dot(word.vector / np.linalg.norm(word.vector), rationale_proto)
+ return cosine_sim
+
+ def _featurize(self, doc):
+ """
+ Take the mean representation, reweighted by the representations of
+ each of the rationale prototypes
+
+ """
+ doc_vects = []
+ for rationale_proto in self.normalized_rationale_proto_by_class.values():
+ doc_vects.append(
+ np.mean(
+ [
+ token.vector * self._rationale_weight(token, rationale_proto)
+ for token in doc if self._valid(token)
+ ],
+ axis=0
+ )
+ )
+ doc_vect = np.mean(doc_vects, axis=0)
+
+ return doc_vect / np.linalg.norm(doc_vect)
+
+
+@Registry.register_experiment(ModeKeys.RATIONALIZED, requirements=[("Featurizer", "PlainTextFeaturizer")])
+class RationaleInformedLRCV(BaseRationaleGridSearch):
+ """
+ Reweight document vectors by their similarity to a rationale vector, predicted by an LR model
+ """
+ def _train_rationale_model(self, docs, rationale_docs, labels=None):
  rationale_vecs = [
  doc.vector / np.linalg.norm(doc.vector) 
  for doc in rationale_docs 
  if doc.has_vector and np.any(np.nonzero(doc.vector))
  ]
  rationale_targets = [1] * len(rationale_vecs)
  background_vecs = [
- doc.vector / np.linalg.norm(doc.vector) 
- for doc in rationale_docs 
+ doc.vector / np.linalg.norm(doc.vector)
+ for doc in docs
  if doc.has_vector and np.any(np.nonzero(doc.vector))
  ]
- background_targets = [0] * len(rationale_vecs)
+ background_targets = [0] * len(background_vecs)
  X = rationale_vecs + background_vecs
  Y = rationale_targets + background_targets