Keep the Misc fields on a CoreLabel

Author: AngledLuffa

itest/src/edu/stanford/nlp/pipeline/CoNLLUReaderITest.java

@@ -52,6 +52,10 @@ public class CoNLLUReaderITest {
  {"sps00", "rg", "da0fp0", "ncfp000", "fp"},
  };
 
+ static final String[][] EXPECTED_MISC = {
+ {null, null, "ArgTem=arg1:tem", null, null, "ArgTem=argM:tmp", "ArgTem=arg1:tem", null, null, null, "ArgTem=argM:loc", "ArgTem=argM:adv", null, null, null, null, null, "ArgTem=arg2:atr", null, "ArgTem=arg0:agt", null},
+ {null, null, null, null, null},
+ };
 
  static final String[][] EXPECTED_FEATS = {
  {
@@ -232,11 +236,12 @@ public void testReadingInCoNLLUFile() throws ClassNotFoundException, IOException
  }
  }
 
- // check the features and that there are no fields currently unaccounted for
+ // check the features, the misc columns, and that there are no fields currently unaccounted for
  for (int i = 0; i < sentences.size(); ++i) {
  CoreMap sentence = sentences.get(i);
  List<CoreLabel> tokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
  assertEquals(EXPECTED_FEATS[i].length, tokens.size());
+ assertEquals(EXPECTED_MISC[i].length, tokens.size());
  for (int j = 0; j < tokens.size(); ++j) {
  CoreLabel token = tokens.get(j);
 
@@ -251,6 +256,15 @@ public void testReadingInCoNLLUFile() throws ClassNotFoundException, IOException
  assertEquals(expected, feats);
  }
 
+ String expectedMisc = EXPECTED_MISC[i][j];
+ if (expectedMisc == null) {
+ assertFalse(token.containsKey(CoreAnnotations.CoNLLUMisc.class));
+ } else {
+ expectedKeys += 1;
+ String misc = token.get(CoreAnnotations.CoNLLUMisc.class).toString();
+ assertEquals(expectedMisc, misc);
+ }
+
  // Some of the AnCora sentences don't have XPOS
  if (token.containsKey(CoreAnnotations.PartOfSpeechAnnotation.class)) {
  expectedKeys += 1;

src/edu/stanford/nlp/pipeline/CoNLLUReader.java

@@ -427,6 +427,7 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
  String spacesBefore = miscKeyValues.get("SpacesBefore");
  if (spacesBefore != null) {
  cl.setBefore(unescapeSpacesAfter(spacesBefore));
+ miscKeyValues.remove("SpacesBefore");
  }
 
  // handle the MWT info and after text
@@ -474,6 +475,21 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
  String spaceAfter = miscToSpaceAfter(miscKeyValues);
  cl.setAfter(spaceAfter);
  }
+ miscKeyValues.remove("SpaceAfter");
+ miscKeyValues.remove("SpacesAfter");
+ if (miscKeyValues.size() > 0) {
+ // rebuild the misc, since we have removed the SpaceAfter, SpacesAfter, and SpacesBefore
+ StringBuilder misc = new StringBuilder();
+ for (Map.Entry<String, String> entry : miscKeyValues.entrySet()) {
+ if (misc.length() > 0) {
+ misc.append("|");
+ }
+ misc.append(entry.getKey());
+ misc.append("=");
+ misc.append(entry.getValue());
+ }
+ cl.set(CoreAnnotations.CoNLLUMisc.class, misc.toString());
+ }
  return cl;
  }