Init

tools/conllu-to-opennlp.py

@@ -0,0 +1,38 @@
+import sys
+limit = int(sys.argv[1]) if len(sys.argv) > 1 else 50000
+out, sent, inside, kept = [], [], False, 0
+def flush():
+    global sent, inside
+    if sent:
+        if inside:
+            sent.append('<END>')
+        out.append(' '.join(sent))
+    sent, inside = [], False
+
+for line in sys.stdin:
+    line = line.rstrip('\n')
+    if not line:
+        flush()
+        if len(out) >= limit:
+            break
+        continue
+    if line.startswith('#'):
+        continue
+    cols = line.split('\t')
+    if len(cols) < 10 or '-' in cols[0]:
+        continue
+    word, tag = cols[1], cols[9]
+    person = tag.endswith('-PER')
+    begins = tag.startswith('Tag=B-PER')
+    if person and (not inside or begins):
+        if inside:
+            sent.append('<END>')
+        sent.append('<START:person>')
+        inside = True
+    elif not person and inside:
+        sent.append('<END>')
+        inside = False
+    sent.append(word)
+flush()
+sys.stderr.write(f"предложений: {len(out)}, с персонами: {sum(1 for l in out if 'START:person' in l)}\n")
+print('\n'.join(out[:limit]))