import sys
import os
from glob import glob
from janome.tokenizer import Tokenizer
def main():
t = Tokenizer()
print('Target Directory:' + os.path.join(input_dir, '*'))
for path in glob(os.path.join(input_dir, 'ja*')):
print('Processing {0}...'.format(path), file=sys.stderr)
with open(path, encoding='utf-8') as file:
txt = file.read()
for token in t.tokenize(txt, stream=True):
print("----" + str(token))
print(token.part_of_speech)
if __name__=='__main__':
main()