ZPar v0.7 compatibility and blank sentence handling

- Modified the Makefile to work with the recently released ZPar v0.7 - Modified python wrappers to intercept blank lines and return empty strings. - Updated wrapper version number to to 0.7.

Author: desilinguist

Makefile

@@ -2,10 +2,10 @@ all: python-zpar
 
 clean:
 rm -rf /tmp/zpar
-rm -f /tmp/zpar.zip
+rm -f /tmp/zpar.tar.gz
 
-python-zpar: clean /tmp/zpar.zip
-unzip -q /tmp/zpar.zip -d /tmp
+python-zpar: clean /tmp/zpar.tar.gz
+tar -C /tmp/zpar -zxf /tmp/zpar.tar.gz --strip-components=1
 cp src/zpar.lib.cpp /tmp/zpar/src/english
 cp src/Makefile.lib.zpar /tmp/zpar
 cp src/Makefile /tmp/zpar
@@ -14,7 +14,8 @@ python-zpar: clean /tmp/zpar.zip
 mkdir -p zpar/dist
 cp /tmp/zpar/dist/zpar.so zpar/dist/
 
-/tmp/zpar.zip:
-wget -N http://sourceforge.net/projects/zpar/files/latest/zpar.zip -O /tmp/zpar.zip
+/tmp/zpar.tar.gz:
+wget -N http://sourceforge.net/projects/zpar/files/latest/zpar.tar.gz -O /tmp/zpar.tar.gz
 touch $@
+mkdir /tmp/zpar
 

conda.yaml

@@ -1,6 +1,6 @@
 package:
  name: python-zpar
- version: "0.6.0"
+ version: "0.7.0"
 
 build:
  number: {{environ.get('BINSTAR_BUILD', 1)}}

setup.py

@@ -82,7 +82,7 @@ def read(fname):
 
 setup(
  name='python-zpar',
- version='0.6',
+ version='0.7',
  description='A Wrapper around the ZPar statistical tagger/parser for English',
  maintainer='Nitin Madnani',
  maintainer_email='nmadnani@ets.org',

src/Makefile

@@ -24,6 +24,15 @@
 #
 #================================================================
 
+# the generic tagger
+TAGGER_IMPL = collins
+
+# the generic depparser
+DEPPARSER_IMPL = arceager
+
+# the generic conparser
+CONPARSER_IMPL = srnew
+
 # choose between agenda, agendachart etc ## NO SPACE AFTER NAME ###
 #
 # agenda: the single agenda method - reproduce paper
@@ -48,18 +57,33 @@ CHINESE_DEPPARSER_LABELED = true
 CHINESE_DEPLABELER_IMPL = naive
 
 # currently support sr implementations
-CHINESE_CONPARSER_IMPL = jcad
+CHINESE_CONPARSER_IMPL = acl13
+CHINESE_CONPARSER_JOINT_OR_CASCADE = JOINT_CONPARSER
 
 # currently support only agenda
 ENGLISH_TAGGER_IMPL = collins
 
 # currently support eisner, covington, nivre, combined implementations
-ENGLISH_DEPPARSER_IMPL =arceager
+ENGLISH_DEPPARSER_IMPL = arceager
 ENGLISH_DEPPARSER_LABELED = true
 ENGLISH_DEPLABELER_IMPL = naive
+
 # currently support sr implementations
 ENGLISH_CONPARSER_IMPL = muhua
 
+# Spanish pos tagger
+SPANISH_TAGGER_IMPL = collins
+
+# Spanish dependency parser
+SPANISH_DEPPARSER_IMPL = arceager
+SPANISH_DEPPARSER_LABELED = true
+SPANISH_DEPLABELER_IMPL = naive
+
+# Spanish annotation. Supported: ES06_DEPENDENCIES, ES09_DEPENDENCIES
+SPANISH_ANNOTATION = ES09_DEPENDENCIES
+
+#no Spanish constituency parser at the moment
+
 #================================================================
 #
 # Debug mode or the run mode (empty)
@@ -78,6 +102,19 @@ DEBUG = -DNDEBUG
 BASE_DIR = .
 include Makefile.common
 
+#================================================================
+#
+# cross platform configurations
+#
+#================================================================
+
+ifeq ($(OS),Windows_NT)
+#use good old GNU mkdir instead of MSDOS mkdir on Windows
+MKDIR=gmkdir -p
+else
+MKDIR=mkdir -p
+endif
+
 #================================================================
 #
 # compiler commands
@@ -100,37 +137,47 @@ LDFLAGS =
 
 # the objects
 LINGUISTICS_OBJECTS = $(OBJECT_DIR)/linguistics/lemma.o $(OBJECT_DIR)/linguistics/conll.o
-OBJECTS = $(OBJECT_DIR)/reader.o $(OBJECT_DIR)/writer.o $(OBJECT_DIR)/options.o $(LINGUISTICS_OBJECTS)
+LEARNING_OBJECTS = $(OBJECT_DIR)/learning/dbn.o
+OBJECTS = $(OBJECT_DIR)/reader.o $(OBJECT_DIR)/writer.o $(OBJECT_DIR)/options.o $(LINGUISTICS_OBJECTS) $(LEARNING_OBJECTS)
 
 $(OBJECT_DIR)/%.o: $(SRC_LIBS)/%.cpp $(SRC_INCLUDES)/%.h
-mkdir -p $(OBJECT_DIR)
-mkdir -p $(OBJECT_DIR)/linguistics
+$(MKDIR) $(OBJECT_DIR)
+$(MKDIR) $(OBJECT_DIR)/linguistics
+$(MKDIR) $(OBJECT_DIR)/learning
 $(CXX) $(CXXFLAGS) -c $< -o $@
 
 all: zpar
 
 # the directories
 $(OBJECT_DIR):
-mkdir -p $(OBJECT_DIR)
+$(MKDIR) $(OBJECT_DIR)
 $(DIST_DIR):
-mkdir -p $(DIST_DIR)
+$(MKDIR) $(DIST_DIR)
 
 # tagger
 SRC_TAGGER = $(SRC_CHINESE)/tagger
 DIST_TAGGER = $(DIST_DIR)/chinese.postagger
 OBJECT_TAGGER = $(OBJECT_DIR)/chinese.postagger
 $(DIST_TAGGER):
-mkdir $(DIST_TAGGER)
+$(MKDIR) $(DIST_TAGGER)
 $(OBJECT_TAGGER):
-mkdir $(OBJECT_TAGGER)
+$(MKDIR) $(OBJECT_TAGGER)
 
 SRC_ENGLISH_TAGGER = $(SRC_COMMON)/tagger
 DIST_ENGLISH_TAGGER = $(DIST_DIR)/english.postagger
 OBJECT_ENGLISH_TAGGER = $(OBJECT_DIR)/english.postagger
 $(DIST_ENGLISH_TAGGER):
-mkdir $(DIST_ENGLISH_TAGGER)
+$(MKDIR) $(DIST_ENGLISH_TAGGER)
 $(OBJECT_ENGLISH_TAGGER):
-mkdir $(OBJECT_ENGLISH_TAGGER)
+$(MKDIR) $(OBJECT_ENGLISH_TAGGER)
+
+SRC_SPANISH_TAGGER = $(SRC_COMMON)/tagger
+DIST_SPANISH_TAGGER = $(DIST_DIR)/spanish.postagger
+OBJECT_SPANISH_TAGGER = $(OBJECT_DIR)/spanish.postagger
+$(DIST_SPANISH_TAGGER):
+$(MKDIR) $(DIST_SPANISH_TAGGER)
+$(OBJECT_SPANISH_TAGGER):
+$(MKDIR) $(OBJECT_SPANISH_TAGGER)
 
 # depparser
 SRC_COMMON_DEPPARSER = $(SRC_COMMON)/depparser
@@ -144,6 +191,8 @@ DIST_DEPPARSER = $(DIST_DIR)/chinese.depparser
 OBJECT_DEPPARSER = $(OBJECT_DIR)/chinese.depparser
 DIST_ENGLISH_DEPPARSER = $(DIST_DIR)/english.depparser
 OBJECT_ENGLISH_DEPPARSER = $(OBJECT_DIR)/english.depparser
+DIST_SPANISH_DEPPARSER = $(DIST_DIR)/spanish.depparser
+OBJECT_SPANISH_DEPPARSER = $(OBJECT_DIR)/spanish.depparser
 
 # deplabeler
 SRC_COMMON_DEPLABELER = $(SRC_COMMON)/deplabeler
@@ -153,14 +202,21 @@ OBJECT_DEPLABELER = $(OBJECT_DIR)/chinese.deplabeler
 SRC_ENGLISH_DEPLABELER = $(SRC_COMMON_DEPLABELER)
 DIST_ENGLISH_DEPLABELER = $(DIST_DIR)/english.deplabeler
 OBJECT_ENGLISH_DEPLABELER = $(OBJECT_DIR)/english.deplabeler
+SRC_SPANISH_DEPLABELER = $(SRC_COMMON_DEPLABELER)
+DIST_SPANISH_DEPLABELER = $(DIST_DIR)/spanish.deplabeler
+OBJECT_SPANISH_DEPLABELER = $(OBJECT_DIR)/spanish.deplabeler
 
 # conparser
 SRC_COMMON_CONPARSER = $(SRC_COMMON)/conparser
 SRC_CHINESE_CONPARSER = $(SRC_COMMON_CONPARSER)
 ifeq ($(CHINESE_CONPARSER_IMPL), jcad)
 SRC_CHINESE_CONPARSER = $(SRC_CHINESE)/conparser
 else
-SRC_CHINESE_CONPARSER = $(SRC_COMMON_CONPARSER)
+ifeq ($(CHINESE_CONPARSER_IMPL), acl13)
+SRC_CHINESE_CONPARSER = $(SRC_CHINESE)/conparser
+else
+SRC_CHINESE_CONPARSER = $(SRC_COMMON_CONPARSER)
+endif
 endif
 SRC_ENGLISH_CONPARSER = $(SRC_COMMON_CONPARSER)
 DIST_CONPARSER = $(DIST_DIR)/chinese.conparser
@@ -174,51 +230,88 @@ OBJECT_ENGLISH_CONPARSER = $(OBJECT_DIR)/english.conparser
 #
 #----------------------------------------------------------------
 
-include Makefile.zpar
-include Makefile.zpar.en
-include Makefile.zpar.ge
-include Makefile.lib.zpar
 
-#----------------------------------------------------------------
-#
-# The sentence boundary detector
-#
-#----------------------------------------------------------------
+ifeq ($(CHINESE_CONPARSER_IMPL), jcad)
+OBJ_CHINESE_CONSTITUENT = $(OBJECT_CONPARSER)/constituent.o $(OBJECT_CONPARSER)/jointconstituent.o
+else
+ifeq ($(CHINESE_CONPARSER_IMPL), acl13)
+OBJ_CHINESE_CONSTITUENT = $(OBJECT_CONPARSER)/constituent.o $(OBJECT_CONPARSER)/jointconstituent.o
+else
+OBJ_CHINESE_CONSTITUENT = $(OBJECT_CONPARSER)/constituent.o
+endif
+endif
 
-include Makefile.doc2snt
+$(DIST_CONPARSER):
+$(MKDIR) $(DIST_CONPARSER)
+$(OBJECT_CONPARSER):
+$(MKDIR) $(OBJECT_CONPARSER)
 
-#----------------------------------------------------------------
-#
-# The segmentor
-#
-#----------------------------------------------------------------
+$(DIST_DEPLABELER):
+$(MKDIR) $(DIST_DEPLABELER)
+$(OBJECT_DEPLABELER):
+$(MKDIR) $(OBJECT_DEPLABELER)
 
-include Makefile.segmentor
+# the flags for train
+ifeq ($(CHINESE_TAGGER_IMPL), segmented) # if segmented
+TAGGER_TRAIN_FLAGS = -DSEGMENTED
+TAGGER_TEST_FLAGS = -DSEGMENTED
+else
+ifeq ($(CHINESE_TAGGER_IMPL), bidirectional) # else if bidirectional
+TAGGER_TRAIN_FLAGS = -DSEGMENTED -DAUTO
+TAGGER_TEST_FLAGS = -DSEGMENTED
+endif
+endif
 
-#----------------------------------------------------------------
-#
-# The pos taggers (Chinese and English)
-#
-#----------------------------------------------------------------
 
-include Makefile.postagger
+ifeq ($(CHINESE_DEPPARSER_LABELED), true)
+CHINESE_DEPPARSER_D = -DLABELED
+endif
 
-#----------------------------------------------------------------
-#
-# The depparsers (Chinese and English)
-#
-#----------------------------------------------------------------
+ifeq ($(ENGLISH_DEPPARSER_LABELED), true)
+ENGLISH_DEPPARSER_D = -DLABELED
+endif
+
+ifeq ($(CHINESE_DEPPARSER_IMPL), combined)
+CHINESE_DEPPARSER_D := $(CHINESE_DEPPARSER_D) -DCOMBINED
+CHINESE_DEPPARSER_IMPL = nivre
+endif
+
+ifeq ($(ENGLISH_DEPPARSER_IMPL), combined)
+ENGLISH_DEPPARSER_D := $(ENGLISH_DEPPARSER_D) -DCOMBINED
+ENGLISH_DEPPARSER_IMPL = nivre
+endif
+
+#====================================================
+
+$(DIST_DEPPARSER):
+$(MKDIR) $(DIST_DEPPARSER)
+$(OBJECT_DEPPARSER):
+$(MKDIR) $(OBJECT_DEPPARSER)
 
-include Makefile.depparser
-include Makefile.deplabeler
+SRC_SEGMENTOR = $(SRC_CHINESE)/segmentor
+DIST_SEGMENTOR = $(DIST_DIR)/segmentor
+OBJECT_SEGMENTOR = $(OBJECT_DIR)/segmentor
+$(DIST_SEGMENTOR):
+$(MKDIR) $(DIST_SEGMENTOR)
+$(OBJECT_SEGMENTOR):
+$(MKDIR) $(OBJECT_SEGMENTOR)
+
+include Makefile.zpar.zh
+include Makefile.zpar.en
+include Makefile.zpar.ge
+include Makefile.zpar.es
+include Makefile.zpar.mvt
+include Makefile.lib.zpar
+
+zpar: zpar.ge
 
 #----------------------------------------------------------------
 #
-# The conparser
+# The sentence boundary detector
 #
 #----------------------------------------------------------------
 
-include Makefile.conparser
+include Makefile.doc2snt
 
 #----------------------------------------------------------------
 #

zpar/DepParser.py

@@ -31,10 +31,16 @@ def __init__(self, modelpath, libptr):
  raise OSError('Cannot find dependency parser model at {}\n'.format(modelpath))
 
  def dep_parse_sentence(self, sentence, tokenize=True):
- zpar_compatible_sentence = sentence.strip() + "\n "
- zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
- parsed_sent = self._dep_parse_sentence(zpar_compatible_sentence, tokenize)
- return parsed_sent.decode('utf-8')
+ if not sentence.strip():
+ # return empty string if the input is empty
+ ans = ""
+ else:
+ zpar_compatible_sentence = sentence.strip() + "\n "
+ zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
+ parsed_sent = self._dep_parse_sentence(zpar_compatible_sentence, tokenize)
+ ans = parsed_sent.decode('utf-8')
+
+ return ans
 
  def dep_parse_file(self, inputfile, outputfile, tokenize=True):
  if os.path.exists(inputfile):

zpar/Parser.py

@@ -31,10 +31,15 @@ def __init__(self, modelpath, libptr):
  raise OSError('Cannot find parser model at {}\n'.format(modelpath))
 
  def parse_sentence(self, sentence, tokenize=True):
- zpar_compatible_sentence = sentence.strip() + "\n "
- zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
- parsed_sent = self._parse_sentence(zpar_compatible_sentence, tokenize)
- return parsed_sent.decode('utf-8')
+ if not sentence.strip():
+ # return empty string if the input is empty
+ ans = ""
+ else:
+ zpar_compatible_sentence = sentence.strip() + "\n "
+ zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
+ parsed_sent = self._parse_sentence(zpar_compatible_sentence, tokenize)
+ ans = parsed_sent.decode('utf-8')
+ return ans
 
  def parse_file(self, inputfile, outputfile, tokenize=True):
  if os.path.exists(inputfile):

zpar/Tagger.py

@@ -30,10 +30,16 @@ def __init__(self, modelpath, libptr):
  raise OSError('Cannot find tagger model at {}\n'.format(modelpath))
 
  def tag_sentence(self, sentence, tokenize=True):
- zpar_compatible_sentence = sentence.strip() + "\n "
- zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
- tagged_sent = self._tag_sentence(zpar_compatible_sentence, tokenize)
- return tagged_sent.decode('utf-8')
+ if not sentence.strip():
+ # return empty string if the input is empty
+ ans = ""
+ else:
+ zpar_compatible_sentence = sentence.strip() + "\n "
+ zpar_compatible_sentence = zpar_compatible_sentence.encode('utf-8')
+ tagged_sent = self._tag_sentence(zpar_compatible_sentence, tokenize)
+ ans = tagged_sent.decode('utf-8')
+
+ return ans
 
  def tag_file(self, inputfile, outputfile, tokenize=True):
  if os.path.exists(inputfile):