Change in review generation and add a new app for name extraction

dashboard/templates/dashboard.html

@@ -23,7 +23,9 @@
                 {{ form|crispy }}
             <input type="hidden" value="{{ review.review_id }}" name="review_id">
             <input class="btn btn-primary ml-2" style="float: right" type="submit", value="Submit"/>
-            <a href="{% url 'predict' review.review_id %}" class="btn btn-info">Analyze</a>
+            {% if review.comment is not None %}
+                <a href="{% url 'predict' review.review_id %}" class="btn btn-info">Analyze</a>
+            {% endif %}
         </form>
         </div>
 

dashboard/views.py

@@ -68,9 +68,10 @@ class UnRepliedReviewList(View):
         now = timezone.now()
         form = ReplyForm()
         date = now - timezone.timedelta(days=30)
-        reviews = Review.objects.filter(reply=None, update_time__gte=date).exclude(comment=None).order_by('update_time')
+        reviews = Review.objects.filter(reply=None, update_time__gte=date)\
+            .exclude(comment=None, star_rating=5).order_by('update_time')
         page = request.GET.get('page', 1)
-        paginator = Paginator(reviews, 30)
+        paginator = Paginator(reviews, 15)
         try:
             reviews = paginator.page(page)
         except PageNotAnInteger:

name_extractor/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

name_extractor/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class NameExtractorConfig(AppConfig):
+    name = 'name_extractor'

name_extractor/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

name_extractor/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

name_extractor/utils.py

@@ -0,0 +1,21 @@
+import re
+from django.conf import settings
+
+
+nlp = settings.MODEL
+
+
+def clean_text(text):
+    # replace some letter in text for getting better performance
+    text = re.sub(r':\s*', ' ', text)
+    text = re.sub(r'&', ',', text)
+    text = re.sub(r'\.*\n\.*', '.', text)
+    text = re.sub(r'[dD][rR](\.|\s*)*', 'Dr. ', text)
+    return text
+
+
+def extract_names(text):
+    text = clean_text(text)
+    doc = nlp(text)
+    names = {ent.text for ent in doc.ents if ent.label_ in ['PERSON', 'ORG']}
+    return list(names)

name_extractor/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

nlu_job/nlu_utils.py

@@ -10,7 +10,7 @@ ner_model = settings.MODEL
 
 def filter_with_last_ten_reviews(location_id, replies):
     replies = list(replies)
-    revs = Review.objects.filter(location_id=location_id).exclude(reply=None).order_by('-update_time')[:12]
+    revs = Review.objects.filter(location_id=location_id).exclude(reply=None).order_by('-update_time')[:10]
     for r in revs:
         s1 = r.reply.replied_text
         for rep in replies:

nlu_job/views.py

@@ -3,6 +3,8 @@ from django.utils import timezone
 from django.shortcuts import redirect
 from review.forms import ReplyForm
 from review.models import Review, CustomReply
+from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+
 
 from .nlu_utils import model_inference, analyze_inference, filter_with_last_ten_reviews
 
@@ -18,7 +20,16 @@ def predict_report(request, review_id):
     now = timezone.now()
     form = ReplyForm()
     date = now - timezone.timedelta(days=30)
-    reviews = Review.objects.filter(reply=None, update_time__gte=date).exclude(comment=None).order_by('update_time')
+    reviews = Review.objects.filter(reply=None, update_time__gte=date)\
+        .exclude(comment=None, star_rating=5).order_by('update_time')
+    page = request.GET.get('page', 1)
+    paginator = Paginator(reviews, 15)
+    try:
+        reviews = paginator.page(page)
+    except PageNotAnInteger:
+        reviews = paginator.page(1)
+    except EmptyPage:
+        reviews = paginator.page(paginator.num_pages)
     replies = {}
     for intent in intents.keys():
         r = CustomReply.objects.filter(reply_category=intent)

review/background_job.py

@@ -1,44 +1,62 @@
 import random
-from threading import Thread
 from time import sleep
 from django.utils import timezone
-from .models import Review, CustomReply
+from .models import Review, CustomReply, Reply
 from nlu_job.nlu_utils import is_a_name
 from review.review_utils import reply_review
+from review.review_utils import populate_reviews
+
+from nameparser import HumanName
 
 
 def un_replied_reviews_with_no_comment():
     now = timezone.now()
     date = now - timezone.timedelta(days=30)
-    reviews = Review.objects.filter(comment=None, reply=None, star_rating=5, update_time__gte=date).order_by('update_time')
+    reviews = Review.objects.filter(comment=None, reply=None, star_rating=5, update_time__gte=date)\
+        .order_by('update_time')
     return reviews
 
 
+def make_parsable_name(name):
+    name = HumanName(name)
+    if name.first == '':
+        if name.title:
+            parsable_name = ' ' + name.title + ' ' + name.last
+        else:
+            parsable_name = ' ' + name.last
+    else:
+        parsable_name = ' ' + name.first.capitalize()
+    return parsable_name
+
+
 def generate_reply(review):
     replies = CustomReply.objects.filter(reply_category='no_comment')
     reply = random.choice(replies)
     name = review.reviewer_name
     if is_a_name(name):
-        parsable_name = ' '+name.split()[0].capitalize()
+        parsable_name = make_parsable_name(name)
     else:
         parsable_name = ''
     replied_text = reply.reply %parsable_name
     return replied_text
 
 
-def task():
-    while True:
-        reviews = un_replied_reviews_with_no_comment()
-        # TODO: reply reviews in background
-        for review in reviews:
-            replied_text = generate_reply(review)
-            response = reply_review(review=review, replied_text=replied_text)
-
-            if response.status_code == 200:
-                print(f'Reply for {review.reviewer_name}\'s comment in {review.location} has successfully done!')
+def reply_uncommented_reviews():
+    reviews = un_replied_reviews_with_no_comment()
+    # TODO: reply reviews in background
+    for review in reviews:
+        replied_text = generate_reply(review)
+        response = reply_review(review=review, replied_text=replied_text)
 
-        sleep(60*random.randint(2, 5))
+        if response.status_code == 200:
+            review_reply = Reply(replied_text=replied_text, create_time=timezone.now())
+            review_reply.save()
+            review.reply = review_reply
+            review.save()
+    sleep(60*random.randint(2, 5))
 
 
-thread = Thread(target=task)
-# thread.start()
+def background_task():
+    populate_reviews()
+    sleep(60 * random.randint(2, 5))
+    reply_uncommented_reviews()

review_automation/settings.py

@@ -30,6 +30,7 @@ INSTALLED_APPS = [
     'dashboard.apps.DashboardConfig',
     'nlu_job.apps.NluJobConfig',
     'analytics.apps.AnalyticsConfig',
+    'name_extractor.apps.NameExtractorConfig',
 
     'crispy_forms',
     'django_crontab',
@@ -132,10 +133,10 @@ NLU_SERVER_URI = 'http://localhost:5005'
 
 # Cron-Jobs of the project
 CRONJOBS = [
-    ('*/10 * * * *', 'review.review_utils.populate_reviews'),
+    ('0 * * * *', 'review.background_job.background_task'),
 ]
 
 # spaCy model
 
 MODEL = spacy.load('en_core_web_md')
-#MODEL = 'NLP NER MODEL'
+# MODEL = 'NLP NER MODEL'