From 48868c687c233ab4a2207962a3101ffe8d708b54 Mon Sep 17 00:00:00 2001
From: Saurabh <MetalBlazer>
Date: Wed, 21 Aug 2019 19:12:26 +0530
Subject: [PATCH 1/8] Similar Questions

---
 .gitignore                                 |  2 +
 static/website/js/custom.js                | 13 ++--
 static/website/templates/new-question.html | 16 ++++-
 website/spam_checker.py                    | 64 +++++++++++++++++
 website/views.py                           | 82 +++++++++++++++++-----
 5 files changed, 152 insertions(+), 25 deletions(-)
 create mode 100644 website/spam_checker.py

diff --git a/.gitignore b/.gitignore
index 2d70242..8d92b6a 100755
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,8 @@ forums/local_settings.py
 static/CACHE/*
 *.py[cod]
 robots.txt
+cuss.csv
+stopwords_English.csv
 # C extensions
 *.so
 
diff --git a/static/website/js/custom.js b/static/website/js/custom.js
index d60e1d5..4089e9b 100755
--- a/static/website/js/custom.js
+++ b/static/website/js/custom.js
@@ -3,6 +3,7 @@ $(document).ready(function() {
     $tutorial = $("#id_tutorial");
     $minute_range = $("#id_minute_range");
     $second_range = $("#id_second_range");
+    $title = $("#id_title")
 		var tutorial = $tutorial.val();
 		var category = $category.val();
 		
@@ -98,15 +99,17 @@ $(document).ready(function() {
         }
     });
     
-    $second_range.change(function() {
+    $title.change(function() {
+        alert("data"+$title.val());
         $.ajax({
             url: "/ajax-similar-questions/",
             type: "POST",
             data: {
-                category: $category.val(),
-                tutorial: $tutorial.val(),
-                minute_range: $minute_range.val(),
-                second_range: $second_range.val()
+                category        : $category.val(),
+                tutorial        : $tutorial.val(),
+                minute_range    : $minute_range.val(),
+                second_range    : $second_range.val(),
+                title           : $title.val(),
             },
             dataType: "html",
             success: function(data) {
diff --git a/static/website/templates/new-question.html b/static/website/templates/new-question.html
index 28fc707..3ac1923 100755
--- a/static/website/templates/new-question.html
+++ b/static/website/templates/new-question.html
@@ -40,7 +40,16 @@ <h4>
         </div>
     </div>
     <hr>
-
+    
+    {% if spam %}
+    <div class="alert alert-danger alert-dismissible  show" style="color: white;font-size: 95%;font-family: sans serif;font-weight: 400;" role="alert">
+<button type="button" class="close" data-dismiss="alert" aria-label="Close">
+  <span aria-hidden="true">&times;</span>
+</button>
+    {{ spam }}
+<br><br><a href="/"> Back to Home</a>
+</div>
+    {% endif %}
     <p>Please enter your question details.</p>
     <div class="row">
         <div class="col-lg-12">
@@ -48,6 +57,7 @@ <h4>
                 <label for="id_title">Title:</label>
                 {% render_field form.title class+="form-control" %} 
             </div>
+            <div id="text"></div>
             <div class="form-group">
                 <label for="id_body">Question:</label>
                 {% render_field form.body class+="form-control" %}
@@ -59,7 +69,7 @@ <h4>
     </form>
 
     <!-- Modal -->
-    <div class="modal fade" id="similarModal" tabindex="-1" role="dialog" aria-labelledby="similarModalLabel" aria-hidden="true">
+    <div class="modal" id="similarModal" tabindex="-1" role="dialog" aria-labelledby="similarModalLabel" aria-hidden="true">
       <div class="modal-dialog">
         <div class="modal-content">
           <div class="modal-header">
@@ -85,5 +95,7 @@ <h4 class="modal-title" id="myModalLabel">Similar Questions</h4>
 {% endblock %}
 
 {% block javascript %}
+
     <script src="{% static 'website/js/custom.js' %}"></script>
+
 {% endblock %}
diff --git a/website/spam_checker.py b/website/spam_checker.py
new file mode 100644
index 0000000..00b5b9e
--- /dev/null
+++ b/website/spam_checker.py
@@ -0,0 +1,64 @@
+#from fuzzywuzzy import fuzz, process
+from website.models import Question, Answer, Notification, AnswerComment
+from spoken_auth.models import FossCategory, TutorialDetails
+from os.path import dirname, curdir, realpath
+import re, csv
+def existing_question(content, category, tutorial):
+    foss_id = FossCategory.objects.filter(foss = category)    
+    tutorial_detail = TutorialDetails.objects.filter(tutorial = tutorial)
+    keywords = TutorialCommonContents.objects.get(tutorial_detail = tutorial_detail)
+    foss_name = slugify_me(str(category))
+    tutorial = slugify_me(str(tutorial))
+    user_ques_parsed = remove_tags(parse_html(str(content)))
+    #user_ques_formatted = vectorizer.fit_transform(user_ques_parsed)
+    questions = Question.objects.filter(category=foss_name, tutorial=tutorial)
+    parsed_question = []
+    log =[]
+    for question in questions:
+        
+        this_question = remove_tags(parse_html(str(question.body)))
+        if len(this_question)>4:
+            parsed_question.append(this_question)
+            log.append((this_question,question.id,fuzz.token_sort_ratio(user_ques_parsed, this_question)))
+            #formatted_question = vectorizer.fit_transform(parsed_question)
+            #print "->",process.extract(user_ques_parsed, this_question)
+    highest = process.extract(user_ques_parsed, parsed_question)
+    
+
+def slugify_me(text):
+    data = text.strip().replace(' ','-')
+    return data
+
+def parse_html(text):
+    soup = BeautifulSoup(text, 'html.parser')
+    return soup.get_text()
+
+def takeThird(elem):
+    return elem[2]
+
+def check_for_cuss(text):
+    dir_path = dirname(realpath(__file__))
+    filepath = dir_path+'/'+'cuss.csv'
+    body = re.split(', | ',text.lower())
+    with open(filepath,'r') as csvfile:
+        spamreader = csv.reader(csvfile, delimiter=',')
+        for spam in spamreader:
+            for word in body:
+                if word in spam:
+                    return True
+            return False
+
+def remove_stop_words(text):
+    title_words = re.split(', | ',text.lower())
+    print "My words are :",title_words
+    important_words = []
+    dir_path = dirname(realpath(__file__))
+    filepath = dir_path+'/'+'stopwords_English.csv'
+    with open(filepath,'r') as csvfile:
+        stop_words = csv.reader(csvfile, delimiter=',').next()
+        print "stop_words",stop_words
+        for a_word in title_words:
+            if a_word not in stop_words :
+                important_words.append(a_word)
+    print "important_words : ",important_words
+    return important_words
diff --git a/website/views.py b/website/views.py
index 386a2f6..ee6d143 100755
--- a/website/views.py
+++ b/website/views.py
@@ -4,6 +4,7 @@
 from django.shortcuts import render, get_object_or_404
 from django.core.context_processors import csrf
 from django.contrib.auth.decorators import login_required
+from django.contrib import messages
 from django.db.models import Q
 from django.core.mail import EmailMultiAlternatives
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
@@ -19,7 +20,7 @@
 from sortable import SortableHeader, get_sorted_list, get_field_index
 from django.db.models import Count
 
-
+from spam_checker import check_for_cuss, remove_stop_words
 User = get_user_model()
 categories = []
 trs = TutorialResources.objects.filter(Q(status=1) | Q(status=2), language__name='English')
@@ -258,9 +259,17 @@ def filter(request, category=None, tutorial=None, minute_range=None, second_rang
 def new_question(request):
     context = {}
     if request.method == 'POST':
-        form = NewQuestionForm(request.POST)
-        if form.is_valid():
+        form = NewQuestionForm(request.POST)        
+        spam_flag = False
+        if form.is_valid():            
             cleaned_data = form.cleaned_data
+            category = request.POST.get('category', None)
+            tutorial = request.POST.get('tutorial', None)
+            content = request.POST['body']
+            title = request.POST['title'] 
+            minute_range = request.POST.get('minute_range', None) 
+            second_range = request.POST.get('second_range', None)
+
             question = Question()
             question.uid = request.user.id
             question.category = cleaned_data['category'].replace(' ', '-')
@@ -269,11 +278,27 @@ def new_question(request):
             question.second_range = cleaned_data['second_range']
             question.title = cleaned_data['title']
             question.body = cleaned_data['body'].encode('unicode_escape')
+
+            if check_for_cuss(cleaned_data['body'].lower().encode('unicode_escape')):
+                              
+                display_message = "You have entered a word which is either harmful or profligate according to our system.\
+                Your question has been added for Admin Review."
+                context['spam'] = display_message                
+                question.status = 0
+                spam_flag = True
+
             question.views = 1
             question.save()
+            context['tut'] = tutorial
+            context['minute_range'] = minute_range            
+            context['second_range'] = second_range
+            context['body'] = content.lstrip().rstrip()
+            context['title2'] = title
+            context['form'] = form
+            context['category'] = category
 
             # Sending email when a new question is asked
-            subject = 'New Forum Question'
+            
             message = """
                 The following new question has been posted in the Spoken Tutorial Forum: <br>
                 Title: <b>{0}</b><br>
@@ -288,16 +313,28 @@ def new_question(request):
                 'http://forums.spoken-tutorial.org/question/' + str(question.id),
                 question.body
             )
-            email = EmailMultiAlternatives(
-                subject, '', 'forums',
-                ['team@spoken-tutorial.org', 'team@fossee.in'],
-                headers={"Content-type": "text/html;charset=iso-8859-1"}
-            )
-            email.attach_alternative(message, "text/html")
-            email.send(fail_silently=True)
-            # End of email send
-
-            return HttpResponseRedirect('/')
+            if spam_flag :
+                subject = 'New Forum Question in Spams'
+                email = EmailMultiAlternatives(
+                    subject, '', 'forums',
+                    ['admin@spoken-tutorial.org', 'nancyvarkey.iitb@gmail.com'],
+                    headers={"Content-type": "text/html;charset=iso-8859-1"}
+                )
+                email.attach_alternative(message, "text/html")
+                email.send(fail_silently=True)
+                # End of email send
+                return render(request, 'website/templates/new-question.html', context)
+            else:
+                subject = 'New Forum Question'
+                email = EmailMultiAlternatives(
+                    subject, '', 'forums',
+                    ['team@spoken-tutorial.org', 'team@fossee.in'],
+                    headers={"Content-type": "text/html;charset=iso-8859-1"}
+                )
+                email.attach_alternative(message, "text/html")
+                email.send(fail_silently=True)
+                # End of email send
+                return HttpResponseRedirect("/")
     else:
         # get values from URL.
         category = request.GET.get('category', None)
@@ -308,7 +345,9 @@ def new_question(request):
         form = NewQuestionForm(category=category, tutorial=tutorial,
                                minute_range=minute_range, second_range=second_range)
         context['category'] = category
-
+        context['tut'] = tutorial
+        context['minute_range'] = minute_range            
+        context['second_range'] = second_range
     context['form'] = form
     context.update(csrf(request))
     return render(request, 'website/templates/new-question.html', context)
@@ -506,11 +545,18 @@ def ajax_similar_questions(request):
     if request.method == 'POST':
         category = request.POST['category']
         tutorial = request.POST['tutorial']
-        # minute_range = request.POST['minute_range']
-        # second_range = request.POST['second_range']
+        category = category.replace(' ','-')
+        tutorial = tutorial.replace(' ','-')
+        title = request.POST['title']
+        important_words = remove_stop_words(title.lower().encode('unicode_escape'))
 
+        questions = Question.objects.none()
         # add more filtering when the forum grows
-        questions = Question.objects.filter(category=category).filter(tutorial=tutorial)
+        for a_word in important_words:
+            required_ques = Question.objects.filter(category=category,tutorial=tutorial,
+                title__icontains=a_word)
+            questions = questions | required_ques
+        print "questions",questions
         context = {
             'questions': questions
         }

From aefad1dc55cb05068bab03195737ce448ee67363 Mon Sep 17 00:00:00 2001
From: Saurabh <MetalBlazer>
Date: Thu, 22 Aug 2019 11:15:05 +0530
Subject: [PATCH 2/8] position of similar questions & min length of title

---
 static/website/templates/new-question.html | 23 ++++++++++++++--------
 website/views.py                           | 20 +++++++++----------
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/static/website/templates/new-question.html b/static/website/templates/new-question.html
index 3ac1923..ee12eba 100755
--- a/static/website/templates/new-question.html
+++ b/static/website/templates/new-question.html
@@ -31,13 +31,7 @@ <h4>
         <div class="col-lg-2 col-md-2 col-sm-2">
             {% render_field form.second_range class+="form-control" disabled="disabled" %} 
         </div>
-        <div class="col-lg-2 col-md-2 col-sm-2">
-            <small><strong>
-            <a id="similar-link" data-toggle="modal" data-target="#similarModal" href="#">
-                0 similar questions
-            </a>
-            </strong></small>
-        </div>
+        
     </div>
     <hr>
     
@@ -51,12 +45,25 @@ <h4>
 </div>
     {% endif %}
     <p>Please enter your question details.</p>
+
     <div class="row">
         <div class="col-lg-12">
             <div class="form-group">
-                <label for="id_title">Title:</label>
+                 <div >
+                <label for="id_title">Title:
+
+           
+            <strong>
+            <a id="similar-link" data-toggle="modal" data-target="#similarModal" href="#">
+                0 similar questions
+            </a>
+            </strong>
+            
+                </label></div>
                 {% render_field form.title class+="form-control" %} 
             </div>
+
+
             <div id="text"></div>
             <div class="form-group">
                 <label for="id_body">Question:</label>
diff --git a/website/views.py b/website/views.py
index ee6d143..7f85825 100755
--- a/website/views.py
+++ b/website/views.py
@@ -548,18 +548,18 @@ def ajax_similar_questions(request):
         category = category.replace(' ','-')
         tutorial = tutorial.replace(' ','-')
         title = request.POST['title']
-        important_words = remove_stop_words(title.lower().encode('unicode_escape'))
-
+        MIN_LENGTH_OF_TITLE = 3
         questions = Question.objects.none()
-        # add more filtering when the forum grows
-        for a_word in important_words:
-            required_ques = Question.objects.filter(category=category,tutorial=tutorial,
-                title__icontains=a_word)
-            questions = questions | required_ques
-        print "questions",questions
+        if len(title) > MIN_LENGTH_OF_TITLE:
+            important_words = remove_stop_words(title.lower().encode('unicode_escape'))
+            # add more filtering when the forum grows
+            for a_word in important_words:
+                required_ques = Question.objects.filter(category=category,tutorial=tutorial,
+                    title__icontains=a_word)
+                questions = questions | required_ques
         context = {
-            'questions': questions
-        }
+                'questions': questions
+            }
         return render(request, 'website/templates/ajax-similar-questions.html', context)
 
 

From fd2368c044c711e69ca688a8b5b319ed2ca49c2f Mon Sep 17 00:00:00 2001
From: Saurabh <MetalBlazer>
Date: Thu, 22 Aug 2019 11:27:18 +0530
Subject: [PATCH 3/8] removed extra alerts

---
 static/website/js/custom.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/static/website/js/custom.js b/static/website/js/custom.js
index 4089e9b..f62cbbf 100755
--- a/static/website/js/custom.js
+++ b/static/website/js/custom.js
@@ -100,7 +100,6 @@ $(document).ready(function() {
     });
     
     $title.change(function() {
-        alert("data"+$title.val());
         $.ajax({
             url: "/ajax-similar-questions/",
             type: "POST",

From dd965b8f63a8eba2f161127356f48034bd2352b8 Mon Sep 17 00:00:00 2001
From: Metalblazer <adhikarysaurabh@gmail.com>
Date: Mon, 30 Sep 2019 16:27:18 +0530
Subject: [PATCH 4/8] minor optimization

---
 website/views.py | 55 ++++++++++++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/website/views.py b/website/views.py
index 7f85825..f5211f0 100755
--- a/website/views.py
+++ b/website/views.py
@@ -264,31 +264,36 @@ def new_question(request):
         if form.is_valid():            
             cleaned_data = form.cleaned_data
             category = request.POST.get('category', None)
-            tutorial = request.POST.get('tutorial', None)
+            #tutorial = request.POST.get('tutorial', None)
+            tutorial = ajax_tutorials(request)
             content = request.POST['body']
             title = request.POST['title'] 
-            minute_range = request.POST.get('minute_range', None) 
-            second_range = request.POST.get('second_range', None)
-
+            #minute_range = request.POST.get('minute_range', None) 
+            #second_range = request.POST.get('second_range', None)
+            duration = ajax_duration(request)
+            minute_range = duration['minutes']
+            second_range = duration['seconds']
             question = Question()
             question.uid = request.user.id
             question.category = cleaned_data['category'].replace(' ', '-')
             question.tutorial = cleaned_data['tutorial'].replace(' ', '-')
-            question.minute_range = cleaned_data['minute_range']
-            question.second_range = cleaned_data['second_range']
+            question.minute_range = minute_range
+            question.second_range = second_range
             question.title = cleaned_data['title']
             question.body = cleaned_data['body'].encode('unicode_escape')
 
             if check_for_cuss(cleaned_data['body'].lower().encode('unicode_escape')):
                               
-                display_message = "You have entered a word which is either harmful or profligate according to our system.\
+                display_message = "You have entered a word which is either harmful or inappropriate according to our system.\
                 Your question has been added for Admin Review."
                 context['spam'] = display_message                
                 question.status = 0
+                question.views = int(question.views) + 1 
                 spam_flag = True
 
             question.views = 1
-            question.save()
+            if not(question.status == 0 and question.views > 1) :
+                question.save()
             context['tut'] = tutorial
             context['minute_range'] = minute_range            
             context['second_range'] = second_range
@@ -315,24 +320,28 @@ def new_question(request):
             )
             if spam_flag :
                 subject = 'New Forum Question in Spams'
-                email = EmailMultiAlternatives(
-                    subject, '', 'forums',
-                    ['admin@spoken-tutorial.org', 'nancyvarkey.iitb@gmail.com'],
-                    headers={"Content-type": "text/html;charset=iso-8859-1"}
-                )
-                email.attach_alternative(message, "text/html")
-                email.send(fail_silently=True)
+                mail_to = 'admin@spoken-tutorial.org,nancyvarkey.iitb@gmail.com'
+                forums_mail(mail_to, subject, message)
+                # email = EmailMultiAlternatives(
+                #     subject, '', 'forums',
+                #     ['admin@spoken-tutorial.org', 'nancyvarkey.iitb@gmail.com'],
+                #     headers={"Content-type": "text/html;charset=iso-8859-1"}
+                # )
+                # email.attach_alternative(message, "text/html")
+                # email.send(fail_silently=True)
                 # End of email send
                 return render(request, 'website/templates/new-question.html', context)
             else:
                 subject = 'New Forum Question'
-                email = EmailMultiAlternatives(
-                    subject, '', 'forums',
-                    ['team@spoken-tutorial.org', 'team@fossee.in'],
-                    headers={"Content-type": "text/html;charset=iso-8859-1"}
-                )
-                email.attach_alternative(message, "text/html")
-                email.send(fail_silently=True)
+                mail_to = 'team@spoken-tutorial.org,team@fossee.in'
+                forums_mail(mail_to, subject, message)
+                # email = EmailMultiAlternatives(
+                #     subject, '', 'forums',
+                #     ['team@spoken-tutorial.org', 'team@fossee.in'],
+                #     headers={"Content-type": "text/html;charset=iso-8859-1"}
+                # )
+                # email.attach_alternative(message, "text/html")
+                # email.send(fail_silently=True)
                 # End of email send
                 return HttpResponseRedirect("/")
     else:
@@ -555,7 +564,7 @@ def ajax_similar_questions(request):
             # add more filtering when the forum grows
             for a_word in important_words:
                 required_ques = Question.objects.filter(category=category,tutorial=tutorial,
-                    title__icontains=a_word)
+                    title__icontains=a_word, status=1)
                 questions = questions | required_ques
         context = {
                 'questions': questions

From 566a77fe4f95f856489a21a299c4cbc130efd837 Mon Sep 17 00:00:00 2001
From: MetalBlazer <adhikarysaurabh@gmail.com>
Date: Thu, 3 Oct 2019 10:59:29 +0530
Subject: [PATCH 5/8] minutes error

---
 website/views.py | 144 ++++++++++++++++++++++++-----------------------
 1 file changed, 75 insertions(+), 69 deletions(-)

diff --git a/website/views.py b/website/views.py
index f5211f0..901c9d1 100755
--- a/website/views.py
+++ b/website/views.py
@@ -264,36 +264,13 @@ def new_question(request):
         if form.is_valid():            
             cleaned_data = form.cleaned_data
             category = request.POST.get('category', None)
-            #tutorial = request.POST.get('tutorial', None)
+            selected_tutorial = request.POST.get('tutorial', None)
             tutorial = ajax_tutorials(request)
             content = request.POST['body']
             title = request.POST['title'] 
-            #minute_range = request.POST.get('minute_range', None) 
-            #second_range = request.POST.get('second_range', None)
-            duration = ajax_duration(request)
-            minute_range = duration['minutes']
-            second_range = duration['seconds']
-            question = Question()
-            question.uid = request.user.id
-            question.category = cleaned_data['category'].replace(' ', '-')
-            question.tutorial = cleaned_data['tutorial'].replace(' ', '-')
-            question.minute_range = minute_range
-            question.second_range = second_range
-            question.title = cleaned_data['title']
-            question.body = cleaned_data['body'].encode('unicode_escape')
-
-            if check_for_cuss(cleaned_data['body'].lower().encode('unicode_escape')):
-                              
-                display_message = "You have entered a word which is either harmful or inappropriate according to our system.\
-                Your question has been added for Admin Review."
-                context['spam'] = display_message                
-                question.status = 0
-                question.views = int(question.views) + 1 
-                spam_flag = True
-
-            question.views = 1
-            if not(question.status == 0 and question.views > 1) :
-                question.save()
+            minute_range = request.POST.get('minute_range', None) 
+            second_range = request.POST.get('second_range', None)
+            body = cleaned_data['body'].encode('unicode_escape')
             context['tut'] = tutorial
             context['minute_range'] = minute_range            
             context['second_range'] = second_range
@@ -301,49 +278,78 @@ def new_question(request):
             context['title2'] = title
             context['form'] = form
             context['category'] = category
-
-            # Sending email when a new question is asked
-            
-            message = """
-                The following new question has been posted in the Spoken Tutorial Forum: <br>
-                Title: <b>{0}</b><br>
-                Category: <b>{1}</b><br>
-                Tutorial: <b>{2}</b><br>
-                Link: <a href="{3}">{3}</a><br>
-                Question: <b>{4}</b><br>
-            """.format(
-                question.title,
-                question.category,
-                question.tutorial,
-                'http://forums.spoken-tutorial.org/question/' + str(question.id),
-                question.body
-            )
-            if spam_flag :
-                subject = 'New Forum Question in Spams'
-                mail_to = 'admin@spoken-tutorial.org,nancyvarkey.iitb@gmail.com'
-                forums_mail(mail_to, subject, message)
-                # email = EmailMultiAlternatives(
-                #     subject, '', 'forums',
-                #     ['admin@spoken-tutorial.org', 'nancyvarkey.iitb@gmail.com'],
-                #     headers={"Content-type": "text/html;charset=iso-8859-1"}
-                # )
-                # email.attach_alternative(message, "text/html")
-                # email.send(fail_silently=True)
-                # End of email send
-                return render(request, 'website/templates/new-question.html', context)
+            repeated_question = Question.objects.filter(category=category,
+                tutorial=selected_tutorial, title=cleaned_data['title'], body=body)
+            if repeated_question.exists():
+                display_message = "You have entered a word which is either harmful or inappropriate according to our system.\
+                Your question has been added for Admin Review."
+                context['spam'] = display_message
+                context['questions'] = repeated_question
+                return render(request, 'website/templates/ajax-similar-questions.html', context)
             else:
-                subject = 'New Forum Question'
-                mail_to = 'team@spoken-tutorial.org,team@fossee.in'
-                forums_mail(mail_to, subject, message)
-                # email = EmailMultiAlternatives(
-                #     subject, '', 'forums',
-                #     ['team@spoken-tutorial.org', 'team@fossee.in'],
-                #     headers={"Content-type": "text/html;charset=iso-8859-1"}
-                # )
-                # email.attach_alternative(message, "text/html")
-                # email.send(fail_silently=True)
-                # End of email send
-                return HttpResponseRedirect("/")
+                question = Question()
+                if check_for_cuss(cleaned_data['body'].lower().encode('unicode_escape')):                              
+                    display_message = "You have entered a word which is either harmful or inappropriate according to our system.\
+                    Your question has been added for Admin Review."
+                    context['spam'] = display_message                
+                    question.status = 0
+                    spam_flag = True
+            
+                question.uid = request.user.id
+                question.category = cleaned_data['category'].replace(' ', '-')
+                question.tutorial = cleaned_data['tutorial'].replace(' ', '-')
+                question.minute_range = minute_range
+                question.second_range = second_range
+                question.title = cleaned_data['title']
+                question.body = body
+                question.views = 1
+                question.save()
+            
+
+                # Sending email when a new question is asked
+                
+                message = """
+                    The following new question has been posted in the Spoken Tutorial Forum: <br>
+                    Title: <b>{0}</b><br>
+                    Category: <b>{1}</b><br>
+                    Tutorial: <b>{2}</b><br>
+                    Link: <a href="{3}">{3}</a><br>
+                    Question: <b>{4}</b><br>
+                """.format(
+                    question.title,
+                    question.category,
+                    question.tutorial,
+                    'http://forums.spoken-tutorial.org/question/' + str(question.id),
+                    question.body
+                )
+                if spam_flag :
+                    subject = 'New Forum Question in Spams'
+                    #mail_to = 'admin@spoken-tutorial.org,nancyvarkey.iitb@gmail.com'
+                    mail_to = 'adhikarysaurabh@gmail.com'
+                    forums_mail(mail_to, subject, message)
+                    # email = EmailMultiAlternatives(
+                    #     subject, '', 'forums',
+                    #     ['admin@spoken-tutorial.org', 'nancyvarkey.iitb@gmail.com'],
+                    #     headers={"Content-type": "text/html;charset=iso-8859-1"}
+                    # )
+                    # email.attach_alternative(message, "text/html")
+                    # email.send(fail_silently=True)
+                    # End of email send
+                    return render(request, 'website/templates/new-question.html', context)
+                else:
+                    subject = 'New Forum Question'
+                    #mail_to = 'team@spoken-tutorial.org,team@fossee.in'
+                    mail_to = 'adhikarysaurabh@gmail.com'
+                    forums_mail(mail_to, subject, message)
+                    # email = EmailMultiAlternatives(
+                    #     subject, '', 'forums',
+                    #     ['team@spoken-tutorial.org', 'team@fossee.in'],
+                    #     headers={"Content-type": "text/html;charset=iso-8859-1"}
+                    # )
+                    # email.attach_alternative(message, "text/html")
+                    # email.send(fail_silently=True)
+                    # End of email send
+                    return HttpResponseRedirect("/")
     else:
         # get values from URL.
         category = request.GET.get('category', None)

From 151685792ffd86e47236cd49c1411fe59e510899 Mon Sep 17 00:00:00 2001
From: Saurabh <MetalBlazer>
Date: Thu, 3 Oct 2019 12:26:21 +0530
Subject: [PATCH 6/8] bug fix

---
 website/forms.py        |  2 +-
 website/spam_checker.py |  2 +-
 website/views.py        | 26 +++++++++++++-------------
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/website/forms.py b/website/forms.py
index c7d5232..8a1cb81 100755
--- a/website/forms.py
+++ b/website/forms.py
@@ -49,7 +49,7 @@ def __init__(self, *args, **kwargs):
             category = args[0]['category']
         if FossCategory.objects.filter(foss=category).exists():
             self.fields['category'].initial = category
-            tutorials = TutorialDetails.objects.using('spoken').filter(foss__foss=category)
+            tutorials = TutorialDetails.objects.using('spoken').filter(foss__foss=category).order_by('level', 'order')
             for tutorial in tutorials:
                 tutorial_choices += ((tutorial.tutorial, tutorial.tutorial),)
             self.fields['tutorial'] = forms.CharField(widget=forms.Select(choices=tutorial_choices))
diff --git a/website/spam_checker.py b/website/spam_checker.py
index 00b5b9e..41992a4 100644
--- a/website/spam_checker.py
+++ b/website/spam_checker.py
@@ -39,7 +39,7 @@ def takeThird(elem):
 def check_for_cuss(text):
     dir_path = dirname(realpath(__file__))
     filepath = dir_path+'/'+'cuss.csv'
-    body = re.split(', | ',text.lower())
+    body = re.split(', | ',text.lower().strip())
     with open(filepath,'r') as csvfile:
         spamreader = csv.reader(csvfile, delimiter=',')
         for spam in spamreader:
diff --git a/website/views.py b/website/views.py
index 901c9d1..a934042 100755
--- a/website/views.py
+++ b/website/views.py
@@ -265,27 +265,27 @@ def new_question(request):
             cleaned_data = form.cleaned_data
             category = request.POST.get('category', None)
             selected_tutorial = request.POST.get('tutorial', None)
-            tutorial = ajax_tutorials(request)
+            
+	    tutorial = request.POST.get('tutorial', None)
             content = request.POST['body']
             title = request.POST['title'] 
             minute_range = request.POST.get('minute_range', None) 
             second_range = request.POST.get('second_range', None)
             body = cleaned_data['body'].encode('unicode_escape')
-            context['tut'] = tutorial
-            context['minute_range'] = minute_range            
-            context['second_range'] = second_range
-            context['body'] = content.lstrip().rstrip()
-            context['title2'] = title
+            context['tutorial'] = tutorial
+            #context['minute_range'] = minute_range            
+            #context['second_range'] = second_range
+            #context['body'] = content.lstrip().rstrip()
+            #context['title2'] = title
             context['form'] = form
             context['category'] = category
-            repeated_question = Question.objects.filter(category=category,
-                tutorial=selected_tutorial, title=cleaned_data['title'], body=body)
+            repeated_question = Question.objects.filter(category=category.replace(' ', '-'),
+                tutorial=selected_tutorial.replace(' ', '-'), title=cleaned_data['title'], body=body)
             if repeated_question.exists():
-                display_message = "You have entered a word which is either harmful or inappropriate according to our system.\
-                Your question has been added for Admin Review."
+                display_message = "The exact same question was asked before hence has not been saved."
                 context['spam'] = display_message
                 context['questions'] = repeated_question
-                return render(request, 'website/templates/ajax-similar-questions.html', context)
+                return HttpResponseRedirect("/")
             else:
                 question = Question()
                 if check_for_cuss(cleaned_data['body'].lower().encode('unicode_escape')):                              
@@ -354,12 +354,12 @@ def new_question(request):
         # get values from URL.
         category = request.GET.get('category', None)
         tutorial = request.GET.get('tutorial', None)
-        minute_range = request.GET.get('minute_range', None)
+	minute_range = request.GET.get('minute_range', None)
         second_range = request.GET.get('second_range', None)
         # pass minute_range and second_range value to NewQuestionForm to populate on select
         form = NewQuestionForm(category=category, tutorial=tutorial,
                                minute_range=minute_range, second_range=second_range)
-        context['category'] = category
+	context['category'] = category
         context['tut'] = tutorial
         context['minute_range'] = minute_range            
         context['second_range'] = second_range

From af998e6326477339f320f6ff53d0e5d8e12c9e56 Mon Sep 17 00:00:00 2001
From: Saurabh <MetalBlazer>
Date: Tue, 28 Apr 2020 10:00:40 +0530
Subject: [PATCH 7/8] python3 changes

---
 website/spam_checker.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/website/spam_checker.py b/website/spam_checker.py
index 120ddc1..8f9940d 100755
--- a/website/spam_checker.py
+++ b/website/spam_checker.py
@@ -49,14 +49,13 @@ def check_for_cuss(text):
             return False
 
 def remove_stop_words(text):
-    title_words = re.split(', | ',text.lower())
+    title_words = re.split(', | ',str(text).lower())
     print("My words are :",title_words)
     important_words = []
     dir_path = dirname(realpath(__file__))
     filepath = dir_path+'/'+'stopwords_English.csv'
     with open(filepath,'r') as csvfile:
-        stop_words = csv.reader(csvfile, delimiter=',').next()
-        print("stop_words",stop_words)
+        stop_words = csv.reader(csvfile, delimiter=',')
         for a_word in title_words:
             if a_word not in stop_words :
                 important_words.append(a_word)

From 659ec130e0a7d1e51f8b6e81c07d608f0ab32d28 Mon Sep 17 00:00:00 2001
From: Saurabh <MetalBlazer>
Date: Tue, 28 Apr 2020 13:26:17 +0530
Subject: [PATCH 8/8] spam working

---
 static/website/templates/new-question.html |  4 +++-
 website/spam_checker.py                    | 10 +++++++++-
 website/views.py                           |  2 +-
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/static/website/templates/new-question.html b/static/website/templates/new-question.html
index fc634f3..79545ed 100755
--- a/static/website/templates/new-question.html
+++ b/static/website/templates/new-question.html
@@ -36,12 +36,14 @@ <h4>
     <hr>
     
     {% if spam %}
-    <div class="alert alert-danger alert-dismissible  show" style="color: white;font-size: 95%;font-family: sans serif;font-weight: 400;" role="alert">
+    <div id="spam" class="alert alert-danger alert-dismissible  show" style="color: white;font-size: 95%;font-family: sans serif;font-weight: 400;" role="alert">
 <button type="button" class="close" data-dismiss="alert" aria-label="Close">
   <span aria-hidden="true">&times;</span>
 </button>
     {{ spam }}
 <br><br><a href="/"> Back to Home</a>
+
+<meta http-equiv="refresh" content="5" />
 </div>
     {% endif %}
     <p>Please enter your question details.</p>
diff --git a/website/spam_checker.py b/website/spam_checker.py
index 8f9940d..42c5ab2 100755
--- a/website/spam_checker.py
+++ b/website/spam_checker.py
@@ -36,10 +36,17 @@ def parse_html(text):
 def takeThird(elem):
     return elem[2]
 
+import re
+def striphtml(data):
+    p = re.compile(r'<.*?>')
+    return p.sub('', data)
+
+
 def check_for_cuss(text):
+    #text = striphtml(text)
     dir_path = dirname(realpath(__file__))
     filepath = dir_path+'/'+'cuss.csv'
-    body = re.split(', | ',text.lower().strip())
+    body = re.split(', | ',str(text).lower().strip())
     with open(filepath,'r') as csvfile:
         spamreader = csv.reader(csvfile, delimiter=',')
         for spam in spamreader:
@@ -49,6 +56,7 @@ def check_for_cuss(text):
             return False
 
 def remove_stop_words(text):
+    #text = striphtml(text)
     title_words = re.split(', | ',str(text).lower())
     print("My words are :",title_words)
     important_words = []
diff --git a/website/views.py b/website/views.py
index c06dd0c..e02669a 100755
--- a/website/views.py
+++ b/website/views.py
@@ -318,7 +318,7 @@ def new_question(request):
                 return HttpResponseRedirect("/")
             else:
                 question = Question()
-                if sc.check_for_cuss(cleaned_data['body'].lower().encode('unicode_escape')):                              
+                if sc.check_for_cuss(cleaned_data['body'].encode('unicode_escape').lower()):                              
                     display_message = "You have entered a word which is either harmful or inappropriate according to our system.\
                     Your question has been added for Admin Review."
                     context['spam'] = display_message