[FEATURE ADDED]:Spam filter module for Spoken tutorials forum

OnlySpam.py

@@ -0,0 +1,44 @@
+import pandas as pd
+from bs4 import BeautifulSoup
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.svm import LinearSVC
+import pickle
+fields = ['Content', 'Spam']
+
+df = pd.read_csv('OnlySpam.csv', usecols=fields, skipinitialspace=True)
+# TAG_RE = re.compile(r'<a.*?>.*?</a>')
+
+
+def remove_tags(text):
+    # string = TAG_RE.sub('hyperlink',text)
+    soup = BeautifulSoup(text, "lxml")
+    if soup.find_all('style'):
+        soup.style.decompose()
+    string = soup.get_text()
+    string = string.replace('&nbsp;', '').replace(
+        '\n', '').replace('\r', '').replace('\t', '')
+    string = ' '.join([w for w in string.split() if len(w) >= 3])
+    return string
+
+
+df['Content'] = df['Content'].apply(remove_tags)
+
+vectorizer = TfidfVectorizer(stop_words='english')
+
+x_train = vectorizer.fit_transform(df['Content'])
+
+model = LinearSVC()
+
+model.fit(x_train, df['Spam'])
+filename = 'spam_model.sav'
+
+pickle.dump(model, open(filename, 'wb'))
+
+
+def predictorspam(comment):
+    simplified = remove_tags(comment)
+    tester = [simplified]
+    contest = vectorizer.transform(tester)
+    load_model = pickle.load(open(filename, 'rb'))
+    a = load_model.predict(contest)
+    return a[0]

README.md

@@ -4,3 +4,10 @@
 1. Create new database and import forums_empty.sql file from 'data' folder.
 2. Create copy the `forums/config.sample.py` to `forums.config.py` and add the values accordingly.
 3. To override any settings, create `forums/local_settings.py` and add the settings there.
+
+# For Spam filter module
+
+1. Install the required dependencies by running `pip install -r mlrequirements.txt`
+2. On future edits in database and/or model scripts rerun the concerned files in the hosted environment.
+    `python Spoken.py`
+    `python OnlySpam.py`

Spoken.py

@@ -0,0 +1,68 @@
+# Libraries
+import pandas as pd
+from bs4 import BeautifulSoup
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.svm import LinearSVC
+# from nltk import word_tokenize
+# from nltk.stem import WordNetLemmatizer
+import pickle
+from imblearn.over_sampling import SMOTE
+fields = ['Content', 'Label']
+
+# Load into dataframe
+df = pd.read_csv('STdataset.csv', skipinitialspace=True, usecols=fields)
+
+# Stripping function
+
+
+def remove_tags(text):
+    soup = BeautifulSoup(text, "lxml")
+    if soup.find_all('style'):
+        soup.style.decompose()
+    string = soup.get_text()
+    string = string.replace(' ', '').replace(
+        '\n', '').replace('\r', '').replace('\t', '')
+    string = ' '.join([w for w in string.split() if len(w) >= 3])
+    return string
+
+
+df['Content'] = df['Content'].apply(remove_tags)
+
+# Lemmatizer
+
+'''
+class LemmaTokenizer(object):
+    def __init__(self):
+        self.wnl = WordNetLemmatizer()
+
+    def __call__(self, doc):
+        return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]
+'''
+
+# Vectorizer
+vectorizer = TfidfVectorizer(stop_words='english')
+
+x = vectorizer.fit_transform(df['Content'])
+
+# Minority oversampling
+sm = SMOTE(random_state=42)
+
+x, y = sm.fit_sample(x, df['Label'])
+
+# Model fitting
+model = LinearSVC(random_state=42, tol=5, fit_intercept=False)
+model.fit(x, y)
+filename = 'tutorial_model.sav'
+pickle.dump(model, open(filename, 'wb'))
+
+# Predictor function
+
+
+def predictor(comment):
+    simplified = remove_tags(comment)
+    tester = [simplified]
+    print(simplified)
+    contest = vectorizer.transform(tester)
+    load_model = pickle.load(open(filename, 'rb'))
+    a = load_model.predict(contest)
+    return a[0]