load frequencies of epub

.gitignore

@@ -160,4 +160,7 @@ cython_debug/
 .idea/
 
 # Project Specific
-scripts/
+scripts/
+.vscode
+.vscode
+data/

setup.py

@@ -31,6 +31,7 @@ setup(
         "nltk",
         "EbookLib",
         "BeautifulSoup",
+        "wordfreq",
     ],
     long_description_content_type='text/markdown',
 )

src/ankimaker/tasks/epub/load_epub.py

@@ -62,14 +62,14 @@ def epub2text(epub_path):
     return ttext
 
 
-def generate_corpus_from_epub_file(input_path):
+def generate_corpus_from_epub_file(input_path, language):
     epub_doc = epub2text(input_path)
-    german_corpus = []
+    corpus = []
-    sw = nltk.corpus.stopwords.words('german')
+    sw = nltk.corpus.stopwords.words(language)
     for content in epub_doc:
-        for w in nltk.word_tokenize(content, language='german'):
+        for w in nltk.word_tokenize(content, language=language):
             w = w.lower()
             if w not in sw and len(w) > 1:
-                german_corpus.append(w)
+                corpus.append(w)
-    return german_corpus
+    return corpus