39 lines
1.1 KiB
Python
39 lines
1.1 KiB
Python
import sys
|
||
limit = int(sys.argv[1]) if len(sys.argv) > 1 else 50000
|
||
out, sent, inside, kept = [], [], False, 0
|
||
def flush():
|
||
global sent, inside
|
||
if sent:
|
||
if inside:
|
||
sent.append('<END>')
|
||
out.append(' '.join(sent))
|
||
sent, inside = [], False
|
||
|
||
for line in sys.stdin:
|
||
line = line.rstrip('\n')
|
||
if not line:
|
||
flush()
|
||
if len(out) >= limit:
|
||
break
|
||
continue
|
||
if line.startswith('#'):
|
||
continue
|
||
cols = line.split('\t')
|
||
if len(cols) < 10 or '-' in cols[0]:
|
||
continue
|
||
word, tag = cols[1], cols[9]
|
||
person = tag.endswith('-PER')
|
||
begins = tag.startswith('Tag=B-PER')
|
||
if person and (not inside or begins):
|
||
if inside:
|
||
sent.append('<END>')
|
||
sent.append('<START:person>')
|
||
inside = True
|
||
elif not person and inside:
|
||
sent.append('<END>')
|
||
inside = False
|
||
sent.append(word)
|
||
flush()
|
||
sys.stderr.write(f"предложений: {len(out)}, с персонами: {sum(1 for l in out if 'START:person' in l)}\n")
|
||
print('\n'.join(out[:limit]))
|