Merge branch 'name-extraction'

dashboard/templates/base.html

@@ -18,19 +18,19 @@
     <header class="site-header">
       <nav class="navbar navbar-expand-md navbar-dark bg-steel fixed-top">
         <div class="container">
-          <a class="navbar-brand mr-4" href="{% url 'un-replied-review' %}">ER-CARE Review</a>
+          <a class="navbar-brand mr-4" href="{% url 'un-replied-review' %}">Byte Trek Online Reputation Manager</a>
           <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarToggle" aria-controls="navbarToggle" aria-expanded="false" aria-label="Toggle navigation">
           <span class="navbar-toggler-icon"></span>
           </button>
           <div class="collapse navbar-collapse" id="navbarToggle">
             <div class="navbar-nav mr-auto">
-              <a class="nav-item nav-link" href="{% url 'un-replied-review' %}">Home</a>
               <a class="nav-item nav-link" href="{% url 'review-list' %}">All Reviews</a>
             </div>
             <!-- Navbar Right Side -->
             <div class="navbar-nav">
                 <a class="nav-item nav-link" href="{% url 'analytics' %}">Analytics</a>
                 <a class="nav-item nav-link" href="{% url 'review-report' %}">App Report</a>
+                <a class="nav-item nav-link" href="{% url 'leader-board' 12345 %}">Leaderboard</a>
             </div>
           </div>
         </div>
@@ -65,7 +65,7 @@
     </main>
     <nav class="navbar navbar-expand-md navbar-dark bg-steel fixed-bottom">
       <div class="container">
-        <span class="navbar-brand">&copy;2020 SignatureCare. - Developed by Byte Trek Ltd. </span>
+        <span class="navbar-brand">&copy;2020 Byte Trek Online Reputation Manager. - Developed by Byte Trek Ltd. </span>
       </div>
     </nav>
     <!-- Optional JavaScript -->

name_extractor/admin.py

@@ -1,3 +1,12 @@
 from django.contrib import admin
+from .models import Staff
 
-# Register your models here.
+
+class StaffAdmin(admin.ModelAdmin):
+    list_display = ('name', 'department', 'location', 'name_mentioned', 'total_units')
+    list_filter = ('location', 'department',)
+    ordering = ['-total_units']
+    search_fields = ['department', 'name']
+
+
+admin.site.register(Staff, StaffAdmin)

name_extractor/migrations/0001_initial.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.0 on 2020-02-09 12:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('gauth', '0008_location_total_review_db'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Staff',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+                ('total_units', models.FloatField()),
+                ('nick_names', models.TextField()),
+                ('location', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gauth.Location')),
+            ],
+        ),
+    ]

name_extractor/migrations/0002_auto_20200213_1122.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0 on 2020-02-13 11:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name_extractor', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='staff',
+            name='nick_names',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='staff',
+            name='total_units',
+            field=models.FloatField(default=0.0),
+        ),
+    ]

name_extractor/migrations/0003_staff_department.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2020-02-13 11:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name_extractor', '0002_auto_20200213_1122'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='staff',
+            name='department',
+            field=models.CharField(default='staff', max_length=255),
+        ),
+    ]

name_extractor/migrations/0004_staff_name_mentioned.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2020-02-19 10:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name_extractor', '0003_staff_department'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='staff',
+            name='name_mentioned',
+            field=models.IntegerField(default=0),
+        ),
+    ]

name_extractor/migrations/0005_auto_20200219_1157.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2020-02-19 11:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name_extractor', '0004_staff_name_mentioned'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='staff',
+            name='total_units',
+            field=models.DecimalField(decimal_places=2, default=0.0, max_digits=5),
+        ),
+    ]

name_extractor/models.py

@@ -1,3 +1,20 @@
 from django.db import models
 
-# Create your models here.
+from gauth.models import Location
+
+
+class Staff(models.Model):
+    name = models.CharField(max_length=255)
+    total_units = models.DecimalField(default=0.0, max_digits=5, decimal_places=2)
+    name_mentioned = models.IntegerField(default=0)
+    location = models.ForeignKey(Location, on_delete=models.CASCADE)
+    department = models.CharField(max_length=255, default='staff')
+    nick_names = models.TextField(null=True, blank=True)
+
+    def __str__(self):
+        return f'{self.name} - {self.location}'
+
+    @property
+    def get_nick_names(self):
+        return self.nick_names.split(',')
+

name_extractor/templates/leaderboard.html

@@ -0,0 +1,33 @@
+{% extends "base.html" %}
+{% block report %}
+  <h2 style="text-align: center">Staff Leader Board</h2>
+<hr>
+  <a href="{% url 'leader-board' 12345 %}"><span class="badge badge-info">All Location</span></a>
+  {% for loc in er_locations %}
+    <a href="{% url 'leader-board' loc.location_id %}"><span class="badge badge-info">{{ loc }}</span></a>
+  {% endfor %}
+<hr>
+    <table class="table mt-2">
+      <thead style="text-align: center">
+        <tr class="table-primary">
+          <th scope="col">Name</th>
+          <th scope="col">Department</th>
+          <th scope="col">Location</th>
+          <th scope="col">Total Mentioned</th>
+          <th scope="col">Total Unit</th>
+        </tr>
+      </thead>
+      <tbody style="text-align: center">
+        {% for staff in staffs %}
+          <tr>
+            <th scope="row">{{ staff.name }}</th>
+            <td>{{ staff.department }}</td>
+            <td>{{ staff.location }}</td>
+            <td>{{ staff.name_mentioned }}</td>
+            <td>{{ staff.total_units }}</td>
+          </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+{% endblock %}
+

name_extractor/urls.py

@@ -0,0 +1,6 @@
+from django.urls import path
+from .views import leader_board
+
+urlpatterns = [
+    path('<location_id>/', leader_board, name='leader-board'),
+]

name_extractor/utils.py

@@ -1,21 +1,73 @@
 import re
+from decimal import Decimal
 from django.conf import settings
-
+from .models import Staff
+from nlu_job.nlu_utils import model_inference
 
 nlp = settings.MODEL
 
+nlu_url = settings.NLU_SERVER_URI
+
+
+def get_all_names(text):
+    res = model_inference(text)
+    entities = res.get('entities')
+    names = {ent.get('value') for ent in entities if ent.get('entity') in ['PERSON', 'ORG']}
+    return names
+
+
+def all_staffs(location):
+    names = []
+    staffs = Staff.objects.filter(location=location)
+    for s in staffs:
+        names.extend(s.get_nick_names)
+    return set(names)
+
 
 def clean_text(text):
     # replace some letter in text for getting better performance
     text = re.sub(r':\s*', ' ', text)
-    text = re.sub(r'&', ',', text)
+    text = re.sub(r'&', ', ', text)
+    text = re.sub(r'/', ', ', text)
     text = re.sub(r'\.*\n\.*', '.', text)
     text = re.sub(r'[dD][rR](\.|\s*)*', 'Dr. ', text)
     return text
 
 
-def extract_names(text):
-    text = clean_text(text)
-    doc = nlp(text)
-    names = {ent.text for ent in doc.ents if ent.label_ in ['PERSON', 'ORG']}
-    return list(names)
+def extract_names(review):
+    text = clean_text(review.comment)
+    # doc = nlp(text)
+    # names = {ent.text for ent in doc.ents if ent.label_ in ['PERSON', 'ORG']}
+
+    names = get_all_names(text)
+    cleaned_names = []
+    for name in names:
+        for n in name.split():
+            cleaned_names.append(n) if n.lower() in all_staffs(review.location) else None
+    return cleaned_names
+
+
+def add_point_to_staff_profile(review):
+    staffs = Staff.objects.filter(location=review.location)
+    names = extract_names(review)
+    point_unit = Decimal(1/len(names)) if not names == [] else 0
+
+    for name in names:
+        for staff in staffs:
+            if name.lower() in staff.get_nick_names:
+                staff.name_mentioned += 1
+                staff.total_units += point_unit
+                staff.save()
+
+
+def make_all_staffs_point_zero():
+    staffs = Staff.objects.all()
+    for s in staffs:
+        s.total_units = 0
+        s.name_mentioned = 0
+        s.save()
+
+
+def extract_names_from_reviews(reviews):
+    for review in reviews:
+        add_point_to_staff_profile(review)

name_extractor/views.py

@@ -1,3 +1,15 @@
 from django.shortcuts import render
+from gauth.models import Location
+from name_extractor.models import Staff
 
-# Create your views here.
+
+def leader_board(request, location_id):
+    if location_id == '12345':
+        staffs = Staff.objects.all().exclude(total_units=0.00).order_by('-total_units')
+    else:
+        staffs = Staff.objects.filter(location_id=location_id).exclude(total_units=0.00).order_by('-total_units')
+
+    locations = Location.objects.all()
+    context = {'er_locations': locations, 'staffs': staffs}
+
+    return render(request, 'leaderboard.html', context=context)

nlu_job/nlu_utils.py

@@ -1,3 +1,4 @@
+import re
 from django.conf import settings
 from requests import post
 import json
@@ -23,8 +24,26 @@ def filter_with_last_ten_reviews(location_id, replies):
     return replies
 
 
+def clean_text(text):
+    # replace some letter in text for getting better performance
+    text = re.sub(r':\s*', ' ', text)
+    text = re.sub(r'&', ', ', text)
+    text = re.sub(r'/', ', ', text)
+    text = re.sub(r'\.*\n\.*', ', ', text)
+    text = re.sub(r'[dD][rR](\.|\s*)*', 'Dr. ', text)
+    emoji_pattern = re.compile("["
+                               u"\U0001F600-\U0001F64F"  # emoticons
+                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
+                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
+                               "]+", flags=re.UNICODE)
+    text = re.sub(emoji_pattern, ' ', text)
+    return text
+
+
 def model_inference(text):
     url = nlu_server_url + '/model/parse'
+    text = clean_text(text)
     payload = {'text': text}
     headers = {'content-type': 'application/json'}
 
@@ -37,11 +56,16 @@ def model_inference(text):
 def is_a_name(name):
     '''
     function that decide whether it is a person name or not
-    :param : name -> a string usually reviewer name
-    :return: Boolean ->  true or false
+    :param -> a string usually reviewer name:
+    :return -> a boolean True/False:
     '''
-    doc = ner_model(name)
-    if doc.ents and doc.ents[0].label_ == 'PERSON':
+
+    response = model_inference(name.title())
+    entities = response.get('entities')
+    if not entities:
+        return False
+    entity = entities[0]
+    if entity.get('entity') == 'PERSON':
         return True
     else:
         return False

review_automation/settings.py

@@ -75,13 +75,12 @@ DATABASES = {
         'NAME': 'review_automation',
         'USER': 'root',
         'PASSWORD': 'sad2002S1',
-        'HOST': 'localhost',
+        'HOST': '10.0.0.17',
         'PORT': '3306',
     }
 }
 
 
-
 # Password validation
 
 AUTH_PASSWORD_VALIDATORS = [
@@ -128,15 +127,15 @@ CLIENT_ID = "174657415928-0bt50gt42pslq47gf21ao67n15rom96r.apps.googleuserconten
 CLIENT_SECRET = "ZXYpt07Su0pW3y3jPGOXY_C_"
 TOKEN_URI = "https://oauth2.googleapis.com/token"
 HOST_URI = "http://127.0.0.1:8000"
-NLU_SERVER_URI = 'http://localhost:5005'
+NLU_SERVER_URI = 'http://10.0.0.33:5005'
 
 
 # Cron-Jobs of the project
 CRONJOBS = [
-    ('0 * * * *', 'review.background_job.background_task'),
+    ('0 6 * * *', 'review.background_job.background_task'),
 ]
 
 # spaCy model
 
-MODEL = spacy.load('en_core_web_md')
-# MODEL = 'NLP NER MODEL'
+# MODEL = spacy.load('en_core_web_md')
+MODEL = 'NLP NER MODEL'

review_automation/urls.py

@@ -12,9 +12,10 @@ urlpatterns = [
     path('nlu/', include('nlu_job.urls')),
     path('dashboard/', include('dashboard.urls')),
     path('analytics/', include('analytics.urls')),
+    path('leaderboard/', include('name_extractor.urls')),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
 
 
-admin.site.site_header = "SignatureCare Review Admin"
-admin.site.site_title = "SignatureCare Admin Portal"
-admin.site.index_title = "Welcome to SignatureCare Review Admin Portal"
+admin.site.site_header = "Byte Trek Online Reputation Manager"
+admin.site.site_title = "BT Online Reputation Manager Admin Portal"
+admin.site.index_title = "Welcome to Byte Trek Online Reputation Manager"