diff --git a/21-constructivist/tf-21.py b/21-constructivist/tf-21.py index 56ab7e6..9403037 100755 --- a/21-constructivist/tf-21.py +++ b/21-constructivist/tf-21.py @@ -1,16 +1,10 @@ #!/usr/bin/env python - import sys, re, operator, string, inspect # # The functions # def extract_words(path_to_file): - """ - Takes a path to a file and returns the non-stop - words, after properly removing nonalphanumeric chars - and normalizing for lower case - """ if type(path_to_file) is not str or not path_to_file: return [] @@ -23,6 +17,11 @@ def extract_words(path_to_file): pattern = re.compile('[\W_]+') word_list = pattern.sub(' ', str_data).lower().split() + return word_list + +def remove_stop_words(word_list): + if type(word_list) is not list: + return [] try: with open('../stop_words.txt') as f: @@ -35,10 +34,6 @@ def extract_words(path_to_file): return [w for w in word_list if not w in stop_words] def frequencies(word_list): - """ - Takes a list of words and returns a dictionary associating - words with frequencies of occurrence - """ if type(word_list) is not list or word_list == []: return {} @@ -51,11 +46,6 @@ def frequencies(word_list): return word_freqs def sort(word_freq): - """ - Takes a dictionary of words and their frequencies - and returns a list of pairs where the entries are - sorted by frequency - """ if type(word_freq) is not dict or word_freq == {}: return [] @@ -65,7 +55,7 @@ def sort(word_freq): # The main function # filename = sys.argv[1] if len(sys.argv) > 1 else "../input.txt" -word_freqs = sort(frequencies(extract_words(filename))) +word_freqs = sort(frequencies(remove_stop_words(extract_words(filename)))) for tf in word_freqs[0:25]: print tf[0], ' - ', tf[1]