gabriel/ankimaker
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

load frequencies of epub

Browse Source
This commit is contained in:
gabriel becker 2023-09-22 22:27:27 +10:00
parent 9aff3d7d43
commit 521c7d694d
5 changed files with 693 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -160,4 +160,7 @@ cython_debug/
.idea/ .idea/
# Project Specific # Project Specific
scripts/ scripts/
.vscode
.vscode
data/

341
notebooks/french_epub_stats.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

341
notebooks/german_epub_stats.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

1
setup.py
View File

@ -31,6 +31,7 @@ setup(
"nltk", "nltk",
"EbookLib", "EbookLib",
"BeautifulSoup", "BeautifulSoup",
"wordfreq",
], ],
long_description_content_type='text/markdown', long_description_content_type='text/markdown',
) )

12
src/ankimaker/tasks/epub/load_epub.py
View File

@ -62,14 +62,14 @@ def epub2text(epub_path):
return ttext return ttext
def generate_corpus_from_epub_file(input_path): def generate_corpus_from_epub_file(input_path, language):
epub_doc = epub2text(input_path) epub_doc = epub2text(input_path)
german_corpus = [] corpus = []
sw = nltk.corpus.stopwords.words('german') sw = nltk.corpus.stopwords.words(language)
for content in epub_doc: for content in epub_doc:
for w in nltk.word_tokenize(content, language='german'): for w in nltk.word_tokenize(content, language=language):
w = w.lower() w = w.lower()
if w not in sw and len(w) > 1: if w not in sw and len(w) > 1:
german_corpus.append(w) corpus.append(w)
return german_corpus return corpus